From 3797753fbcc2f68a4b37e1ac67ff710c2b8dbd27 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> Date: Thu, 20 Aug 2026 06:14:04 +0000 Subject: [PATCH] feat(tts): support multi-reference personalities Saved profiles previously resolved to one audio path and transcript, so cloning backends could not use several examples of one personality. Store ordered audio and transcript pairs while preserving the legacy first-reference fields. Fish Speech and audio.cpp receive all pairs, including on distributed workers. Other backends retain their single-reference behavior. Assisted-by: Codex:gpt-5 --- backend/cpp/audio-cpp/generation_request.cpp | 3 +- .../audio-cpp/generation_request_ctest.cpp | 8 + backend/python/fish-speech/backend.py | 22 ++- core/http/endpoints/localai/tts.go | 23 ++- core/http/endpoints/localai/voice_profiles.go | 71 ++++--- .../endpoints/localai/voice_profiles_test.go | 20 ++ core/http/react-ui/e2e/voice-library.spec.js | 4 +- .../react-ui/public/locales/en/media.json | 37 ++-- .../react-ui/src/pages/VoiceProfileCreate.jsx | 42 ++++- core/services/nodes/file_staging_client.go | 39 ++++ core/services/nodes/file_staging_tts_test.go | 15 ++ core/services/voiceprofile/store.go | 174 ++++++++++++------ core/services/voiceprofile/store_test.go | 23 +++ docs/content/features/text-to-audio.md | 18 +- pkg/mcp/localaitools/dto.go | 18 +- pkg/mcp/localaitools/inproc/client.go | 23 ++- swagger/docs.go | 17 ++ swagger/swagger.json | 17 ++ swagger/swagger.yaml | 11 ++ 19 files changed, 465 insertions(+), 120 deletions(-) diff --git a/backend/cpp/audio-cpp/generation_request.cpp b/backend/cpp/audio-cpp/generation_request.cpp index f74cb1a210d7..805bf35d8d8a 100644 --- a/backend/cpp/audio-cpp/generation_request.cpp +++ b/backend/cpp/audio-cpp/generation_request.cpp @@ -22,7 +22,8 @@ bool voice_is_reference_file(const std::string &voice) { RequestShape build_tts_shape(const backend::TTSRequest &request) { RequestShape shape; - shape.has_voice_reference = voice_is_reference_file(request.voice()); + shape.has_voice_reference = voice_is_reference_file(request.voice()) || + request.params().find("multi_reference_cond") != request.params().end(); // !empty() as well as has_instructions(), and it must match the guard in // build_tts_request: a request whose instructions are an empty string // carries no style condition, so telling routing to prefer VoiceDesign for diff --git a/backend/cpp/audio-cpp/generation_request_ctest.cpp b/backend/cpp/audio-cpp/generation_request_ctest.cpp index 507b2054de72..e9b5d51069bc 100644 --- a/backend/cpp/audio-cpp/generation_request_ctest.cpp +++ b/backend/cpp/audio-cpp/generation_request_ctest.cpp @@ -128,6 +128,14 @@ static void test_tts_shape() { check(!shape.has_voice_reference, "shape: a preset name is not a voice reference"); } + { + backend::TTSRequest request; + (*request.mutable_params())["multi_reference_cond"] = + R"([{"audio":"one.wav","text":"one"}])"; + const auto shape = build_tts_shape(request); + check(shape.has_voice_reference, + "shape: multi-reference conditioning is a voice reference"); + } { backend::TTSRequest request; request.set_voice(dir.string()); diff --git a/backend/python/fish-speech/backend.py b/backend/python/fish-speech/backend.py index 138396599a63..6dd46c4815c1 100644 --- a/backend/python/fish-speech/backend.py +++ b/backend/python/fish-speech/backend.py @@ -326,11 +326,29 @@ def TTS(self, request, context): max_new_tokens = self.options.get("max_new_tokens", 1024) chunk_length = self.options.get("chunk_length", 200) - # Build references list for voice cloning + # Build references list for voice cloning. Saved LocalAI + # personalities use the same ordered JSON shape as audio.cpp. references = [] voice_name = request.voice if request.voice else None - if voice_name and os.path.isfile(voice_name): + multi_reference_cond = request.params.get("multi_reference_cond", "") if hasattr(request, "params") else "" + if multi_reference_cond: + reference_entries = json.loads(multi_reference_cond) + if not isinstance(reference_entries, list) or not reference_entries: + raise ValueError("multi_reference_cond must be a non-empty JSON array") + for entry in reference_entries: + if not isinstance(entry, dict) or not entry.get("audio") or not entry.get("text"): + raise ValueError("multi_reference_cond entries require audio and text") + ref_audio_path = self._get_ref_audio_path(entry["audio"]) + with open(ref_audio_path, "rb") as f: + audio_bytes = f.read() + references.append(ServeReferenceAudio(audio=audio_bytes, text=entry["text"])) + print( + f"[INFO] Using {len(references)} per-request reference audios", + file=sys.stderr, + ) + + elif voice_name and os.path.isfile(voice_name): ref_audio_path = self._get_ref_audio_path(voice_name) with open(ref_audio_path, "rb") as f: audio_bytes = f.read() diff --git a/core/http/endpoints/localai/tts.go b/core/http/endpoints/localai/tts.go index 16b49a91267f..f8d6c558fdc6 100644 --- a/core/http/endpoints/localai/tts.go +++ b/core/http/endpoints/localai/tts.go @@ -1,6 +1,7 @@ package localai import ( + "encoding/json" "errors" "fmt" "net/http" @@ -67,7 +68,7 @@ func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig if profiles == nil { return echo.NewHTTPError(http.StatusInternalServerError, "voice profile store is unavailable") } - profile, referencePath, release, err := profiles.LeaseAudio(c.Request().Context(), profileID) + profile, referencePaths, release, err := profiles.LeaseAudios(c.Request().Context(), profileID) if err != nil { if errors.Is(err, voiceprofile.ErrNotFound) { return echo.NewHTTPError(http.StatusNotFound, "voice profile not found") @@ -75,7 +76,7 @@ func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig return fmt.Errorf("resolve voice profile: %w", err) } defer release() - cfg.Voice = referencePath + cfg.Voice = referencePaths[0] if cfg.Language == "" && profile.Language != "" { cfg.Language = profile.Language } @@ -83,6 +84,20 @@ func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig input.Params = make(map[string]string) } input.Params["ref_text"] = profile.Transcript + if supportsMultipleVoiceReferences(cfg.Backend) && len(referencePaths) > 1 { + references := make([]map[string]string, 0, len(referencePaths)) + for index, path := range referencePaths { + references = append(references, map[string]string{"audio": path, "text": profile.References[index].Transcript}) + } + encoded, err := json.Marshal(references) + if err != nil { + return fmt.Errorf("encode voice profile references: %w", err) + } + input.Params["multi_reference_cond"] = string(encoded) + if cfg.Backend == "audio-cpp" { + cfg.Voice = "" + } + } xlog.Debug("Resolved saved voice profile", "id", profile.ID, "model", input.Model) } } @@ -138,3 +153,7 @@ func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig return c.Attachment(filePath, filepath.Base(filePath)) } } + +func supportsMultipleVoiceReferences(backendName string) bool { + return backendName == "fish-speech" || backendName == "audio-cpp" +} diff --git a/core/http/endpoints/localai/voice_profiles.go b/core/http/endpoints/localai/voice_profiles.go index 38990d212cb7..d2f19eb4a568 100644 --- a/core/http/endpoints/localai/voice_profiles.go +++ b/core/http/endpoints/localai/voice_profiles.go @@ -14,7 +14,7 @@ import ( "github.com/mudler/xlog" ) -const maxVoiceProfileJSONBytes = voiceprofile.MaxAudioBytes*4/3 + 2*1024*1024 +const maxVoiceProfileJSONBytes = voiceprofile.MaxAudioBytes*voiceprofile.MaxReferences*4/3 + 2*1024*1024 // VoiceProfileListResponse is returned by the profile library endpoint. type VoiceProfileListResponse struct { @@ -25,12 +25,18 @@ type VoiceProfileListResponse struct { // It is primarily used by the LocalAI admin MCP client; the browser sends a // multipart audio field to avoid base64 overhead. type CreateVoiceProfileRequest struct { - Name string `json:"name"` - Description string `json:"description,omitempty"` - Language string `json:"language,omitempty"` - Transcript string `json:"transcript"` - AudioBase64 string `json:"audio_base64"` - ConsentConfirmed bool `json:"consent_confirmed"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Language string `json:"language,omitempty"` + Transcript string `json:"transcript"` + AudioBase64 string `json:"audio_base64"` + ConsentConfirmed bool `json:"consent_confirmed"` + References []CreateVoiceProfileReferenceRequest `json:"references,omitempty"` +} + +type CreateVoiceProfileReferenceRequest struct { + Transcript string `json:"transcript"` + AudioBase64 string `json:"audio_base64"` } func voiceProfileError(code int, message string) schema.ErrorResponse { @@ -118,31 +124,39 @@ func CreateVoiceProfileEndpoint(store *voiceprofile.Store) echo.HandlerFunc { contentType := c.Request().Header.Get(echo.HeaderContentType) if strings.HasPrefix(contentType, echo.MIMEMultipartForm) { - c.Request().Body = http.MaxBytesReader(c.Response().Writer, c.Request().Body, voiceprofile.MaxAudioBytes+2*1024*1024) - fileHeader, err := c.FormFile("audio") + c.Request().Body = http.MaxBytesReader(c.Response().Writer, c.Request().Body, voiceprofile.MaxAudioBytes*voiceprofile.MaxReferences+2*1024*1024) + form, err := c.MultipartForm() if err != nil { if isRequestBodyTooLarge(err) { return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge) } return writeVoiceProfileError(c, fmt.Errorf("%w: audio is required", voiceprofile.ErrInvalidInput)) } - if fileHeader.Size > voiceprofile.MaxAudioBytes { - return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge) + files := form.File["audio"] + transcripts := form.Value["transcript"] + if len(files) == 0 || len(files) != len(transcripts) || len(files) > voiceprofile.MaxReferences { + return writeVoiceProfileError(c, fmt.Errorf("%w: each audio needs one transcript", voiceprofile.ErrInvalidInput)) } - audio, err := fileHeader.Open() - if err != nil { - return writeVoiceProfileError(c, fmt.Errorf("open uploaded audio: %w", err)) + references := make([]voiceprofile.ReferenceInput, 0, len(files)) + for index, fileHeader := range files { + if fileHeader.Size > voiceprofile.MaxAudioBytes { + return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge) + } + audio, err := fileHeader.Open() + if err != nil { + return writeVoiceProfileError(c, fmt.Errorf("open uploaded audio: %w", err)) + } + defer func() { _ = audio.Close() }() + references = append(references, voiceprofile.ReferenceInput{Transcript: transcripts[index], Audio: audio}) } - defer func() { _ = audio.Close() }() consent, _ := strconv.ParseBool(c.FormValue("consent_confirmed")) - profile, err := store.Create(c.Request().Context(), voiceprofile.CreateInput{ + profile, err := store.CreateWithReferences(c.Request().Context(), voiceprofile.CreateInput{ Name: c.FormValue("name"), Description: c.FormValue("description"), Language: c.FormValue("language"), - Transcript: c.FormValue("transcript"), ConsentConfirmed: consent, - }, audio) + }, references) if err != nil { return writeVoiceProfileError(c, err) } @@ -157,17 +171,26 @@ func CreateVoiceProfileEndpoint(store *voiceprofile.Store) echo.HandlerFunc { } return writeVoiceProfileError(c, fmt.Errorf("%w: invalid JSON body", voiceprofile.ErrInvalidInput)) } - if base64.StdEncoding.DecodedLen(len(request.AudioBase64)) > int(voiceprofile.MaxAudioBytes) { - return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge) + referenceRequests := request.References + if len(referenceRequests) == 0 { + referenceRequests = []CreateVoiceProfileReferenceRequest{{Transcript: request.Transcript, AudioBase64: request.AudioBase64}} + } + if len(referenceRequests) > voiceprofile.MaxReferences { + return writeVoiceProfileError(c, voiceprofile.ErrInvalidInput) + } + references := make([]voiceprofile.ReferenceInput, 0, len(referenceRequests)) + for _, reference := range referenceRequests { + if base64.StdEncoding.DecodedLen(len(reference.AudioBase64)) > int(voiceprofile.MaxAudioBytes) { + return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge) + } + references = append(references, voiceprofile.ReferenceInput{Transcript: reference.Transcript, Audio: base64.NewDecoder(base64.StdEncoding, strings.NewReader(reference.AudioBase64))}) } - audio := base64.NewDecoder(base64.StdEncoding, strings.NewReader(request.AudioBase64)) - profile, err := store.Create(c.Request().Context(), voiceprofile.CreateInput{ + profile, err := store.CreateWithReferences(c.Request().Context(), voiceprofile.CreateInput{ Name: request.Name, Description: request.Description, Language: request.Language, - Transcript: request.Transcript, ConsentConfirmed: request.ConsentConfirmed, - }, audio) + }, references) if err != nil { return writeVoiceProfileError(c, err) } diff --git a/core/http/endpoints/localai/voice_profiles_test.go b/core/http/endpoints/localai/voice_profiles_test.go index c913f8355d37..e0425cb2141a 100644 --- a/core/http/endpoints/localai/voice_profiles_test.go +++ b/core/http/endpoints/localai/voice_profiles_test.go @@ -121,6 +121,26 @@ var _ = Describe("Voice profile endpoints", func() { Expect(recorder.Code).To(Equal(http.StatusCreated), recorder.Body.String()) }) + It("creates a profile with ordered JSON references", func() { + payload, err := json.Marshal(CreateVoiceProfileRequest{ + Name: "Personality", ConsentConfirmed: true, + References: []CreateVoiceProfileReferenceRequest{ + {Transcript: "First reference.", AudioBase64: base64.StdEncoding.EncodeToString(voiceProfileWAV(time.Second))}, + {Transcript: "Second reference.", AudioBase64: base64.StdEncoding.EncodeToString(voiceProfileWAV(2 * time.Second))}, + }, + }) + Expect(err).NotTo(HaveOccurred()) + request := httptest.NewRequest(http.MethodPost, "/api/voice-profiles", bytes.NewReader(payload)) + request.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + recorder := httptest.NewRecorder() + e.ServeHTTP(recorder, request) + Expect(recorder.Code).To(Equal(http.StatusCreated), recorder.Body.String()) + var created voiceprofile.Profile + Expect(json.Unmarshal(recorder.Body.Bytes(), &created)).To(Succeed()) + Expect(created.References).To(HaveLen(2)) + Expect(created.Transcript).To(Equal("First reference.")) + }) + It("rejects creation without explicit consent", func() { body, contentType := voiceProfileMultipart("false") request := httptest.NewRequest(http.MethodPost, "/api/voice-profiles", body) diff --git a/core/http/react-ui/e2e/voice-library.spec.js b/core/http/react-ui/e2e/voice-library.spec.js index 07b82ccb4920..0ef49f9e048d 100644 --- a/core/http/react-ui/e2e/voice-library.spec.js +++ b/core/http/react-ui/e2e/voice-library.spec.js @@ -60,7 +60,7 @@ async function mockVoiceAPIs(page) { return state } -test.describe('Voice Library', () => { +test.describe('Personality Library', () => { let apiState test.beforeEach(async ({ page }) => { @@ -69,7 +69,7 @@ test.describe('Voice Library', () => { test('renders the library-first master/detail view', async ({ page }) => { await page.goto('/app/voice-library') - await expect(page.getByRole('heading', { name: /Voice Library/i })).toBeVisible() + await expect(page.getByRole('heading', { name: /Personality Library/i })).toBeVisible() await expect(page.locator('.voice-row', { hasText: 'Documentary narrator' })).toBeVisible() await expect(page.locator('.voice-library-detail')).toContainText('The exact words spoken in this reference.') await expect(page.locator('.voice-library-detail')).toContainText('Consent confirmed') diff --git a/core/http/react-ui/public/locales/en/media.json b/core/http/react-ui/public/locales/en/media.json index 1b392d9d5c7b..af93d06f5b0c 100644 --- a/core/http/react-ui/public/locales/en/media.json +++ b/core/http/react-ui/public/locales/en/media.json @@ -172,25 +172,25 @@ }, "voiceLibrary": { "cloningReady": "Voice cloning", - "loading": "Loading saved voices…", + "loading": "Loading saved personalities…", "modelDefault": "Use model default", - "empty": "No saved voices yet.", + "empty": "No saved personalities yet.", "create": "Create one", - "manage": "Manage Voice Library", + "manage": "Manage Personality Library", "namedVoiceHint": "This model does not accept reference-audio profiles. You can still enter a named speaker supported by its backend." } }, "voiceLibrary": { - "title": "Voice Library", - "subtitle": "Create and manage reusable reference voices for every installed model that supports voice cloning.", - "loading": "Loading saved voices…", - "listLabel": "Saved voice profiles", + "title": "Personality Library", + "subtitle": "Create and manage reusable voice personalities with one or more reference recordings.", + "loading": "Loading saved personalities…", + "listLabel": "Saved voice personalities", "status": { "ready": "Ready" }, "summary": { "label": "Voice library status", - "profiles": "saved voices", + "profiles": "saved personalities", "modelsReady_one": "{{count}} compatible model ready", "modelsReady_other": "{{count}} compatible models ready", "noModels": "No compatible model installed" @@ -204,8 +204,8 @@ "allLanguages": "All languages" }, "empty": { - "title": "Your voice library is empty", - "body": "Record or upload one consented reference clip, add its exact transcript, and reuse it across compatible TTS models." + "title": "Your personality library is empty", + "body": "Record or upload consented reference clips, add their exact transcripts, and reuse the personality across compatible TTS models." }, "noResults": { "title": "No voices match these filters", @@ -251,12 +251,12 @@ "endpointNote": "The same model, input, and voice fields also work with POST /tts." }, "actions": { - "create": "Create voice", - "createFirst": "Create your first voice", + "create": "Create personality", + "createFirst": "Create your first personality", "retry": "Try again", "clearFilters": "Clear filters", "useInTTS": "Use in Text to Speech", - "delete": "Delete voice", + "delete": "Delete personality", "installModel": "Install a compatible model", "browseModels": "Browse all models" }, @@ -274,9 +274,14 @@ } }, "voiceCreate": { - "eyebrow": "Voice Library", - "title": "Create a reusable voice", - "subtitle": "Add one clean reference clip and its exact transcript. LocalAI handles the backend-specific cloning parameters at generation time.", + "eyebrow": "Personality Library", + "title": "Create a reusable personality", + "subtitle": "Create a reusable personality from one or more clean reference clips and their exact transcripts.", + "references": { + "add": "Add another reference", + "additional": "Reference {{number}}", + "remove": "Remove" + }, "sections": { "reference": { "title": "Reference audio", diff --git a/core/http/react-ui/src/pages/VoiceProfileCreate.jsx b/core/http/react-ui/src/pages/VoiceProfileCreate.jsx index 14d1caa02c66..502447ef9b66 100644 --- a/core/http/react-ui/src/pages/VoiceProfileCreate.jsx +++ b/core/http/react-ui/src/pages/VoiceProfileCreate.jsx @@ -66,6 +66,7 @@ export default function VoiceProfileCreate() { const [description, setDescription] = useState('') const [language, setLanguage] = useState('') const [transcript, setTranscript] = useState('') + const [additionalReferences, setAdditionalReferences] = useState([]) const [consent, setConsent] = useState(false) const [submitting, setSubmitting] = useState(false) @@ -75,7 +76,8 @@ export default function VoiceProfileCreate() { const durationValid = audio?.duration >= 1 && audio?.duration <= 120 const durationRecommended = audio?.duration >= 6 && audio?.duration <= 30 - const formReady = !!audio && durationValid && name.trim() && transcript.trim() && consent && !audioProcessing + const additionalReady = additionalReferences.every(reference => reference.audio && reference.transcript.trim()) + const formReady = !!audio && durationValid && name.trim() && transcript.trim() && additionalReady && consent && !audioProcessing const readiness = useMemo(() => ({ audio: !!audio && durationValid, @@ -108,6 +110,23 @@ export default function VoiceProfileCreate() { } } + const handleAdditionalAudio = async (index, sample) => { + if (!sample) { + setAdditionalReferences(current => current.map((reference, itemIndex) => itemIndex === index ? { ...reference, audio: null } : reference)) + return + } + setAudioProcessing(true) + try { + const normalized = await normalizeAudioSample(sample) + if (normalized.duration < 1 || normalized.duration > 120) throw new Error(t('voiceCreate.audio.durationError')) + setAdditionalReferences(current => current.map((reference, itemIndex) => itemIndex === index ? { ...reference, audio: normalized } : reference)) + } catch (err) { + addToast(err.message || t('voiceCreate.audio.decodeError'), 'error') + } finally { + setAudioProcessing(false) + } + } + const submit = async (event) => { event.preventDefault() if (!formReady) return @@ -120,6 +139,10 @@ export default function VoiceProfileCreate() { formData.append('transcript', transcript.trim()) formData.append('consent_confirmed', 'true') formData.append('audio', audio.blob, 'reference.wav') + additionalReferences.forEach((reference, index) => { + formData.append('transcript', reference.transcript.trim()) + formData.append('audio', reference.audio.blob, `reference-${index + 2}.wav`) + }) const profile = await voiceProfilesApi.create(formData) addToast(t('voiceCreate.toasts.created', { name: profile.name }), 'success') navigate(`/app/voice-library?selected=${encodeURIComponent(profile.id)}`) @@ -133,7 +156,7 @@ export default function VoiceProfileCreate() { return (
)} + {additionalReferences.map((reference, index) => ( +
+
+ {t('voiceCreate.references.additional', { number: index + 2 })} + +
+ handleAdditionalAudio(index, sample)} maxBytes={MAX_AUDIO_BYTES} preferBlob idPrefix={`voice-profile-${index + 2}`} /> + +