diff --git a/backend/cpp/audio-cpp/generation_request.cpp b/backend/cpp/audio-cpp/generation_request.cpp index f74cb1a210d7..805bf35d8d8a 100644 --- a/backend/cpp/audio-cpp/generation_request.cpp +++ b/backend/cpp/audio-cpp/generation_request.cpp @@ -22,7 +22,8 @@ bool voice_is_reference_file(const std::string &voice) { RequestShape build_tts_shape(const backend::TTSRequest &request) { RequestShape shape; - shape.has_voice_reference = voice_is_reference_file(request.voice()); + shape.has_voice_reference = voice_is_reference_file(request.voice()) || + request.params().find("multi_reference_cond") != request.params().end(); // !empty() as well as has_instructions(), and it must match the guard in // build_tts_request: a request whose instructions are an empty string // carries no style condition, so telling routing to prefer VoiceDesign for diff --git a/backend/cpp/audio-cpp/generation_request_ctest.cpp b/backend/cpp/audio-cpp/generation_request_ctest.cpp index 507b2054de72..e9b5d51069bc 100644 --- a/backend/cpp/audio-cpp/generation_request_ctest.cpp +++ b/backend/cpp/audio-cpp/generation_request_ctest.cpp @@ -128,6 +128,14 @@ static void test_tts_shape() { check(!shape.has_voice_reference, "shape: a preset name is not a voice reference"); } + { + backend::TTSRequest request; + (*request.mutable_params())["multi_reference_cond"] = + R"([{"audio":"one.wav","text":"one"}])"; + const auto shape = build_tts_shape(request); + check(shape.has_voice_reference, + "shape: multi-reference conditioning is a voice reference"); + } { backend::TTSRequest request; request.set_voice(dir.string()); diff --git a/backend/python/fish-speech/backend.py b/backend/python/fish-speech/backend.py index 138396599a63..6dd46c4815c1 100644 --- a/backend/python/fish-speech/backend.py +++ b/backend/python/fish-speech/backend.py @@ -326,11 +326,29 @@ def TTS(self, request, context): max_new_tokens = self.options.get("max_new_tokens", 1024) chunk_length = self.options.get("chunk_length", 200) - # Build references list for voice cloning + # Build references list for voice cloning. Saved LocalAI + # personalities use the same ordered JSON shape as audio.cpp. references = [] voice_name = request.voice if request.voice else None - if voice_name and os.path.isfile(voice_name): + multi_reference_cond = request.params.get("multi_reference_cond", "") if hasattr(request, "params") else "" + if multi_reference_cond: + reference_entries = json.loads(multi_reference_cond) + if not isinstance(reference_entries, list) or not reference_entries: + raise ValueError("multi_reference_cond must be a non-empty JSON array") + for entry in reference_entries: + if not isinstance(entry, dict) or not entry.get("audio") or not entry.get("text"): + raise ValueError("multi_reference_cond entries require audio and text") + ref_audio_path = self._get_ref_audio_path(entry["audio"]) + with open(ref_audio_path, "rb") as f: + audio_bytes = f.read() + references.append(ServeReferenceAudio(audio=audio_bytes, text=entry["text"])) + print( + f"[INFO] Using {len(references)} per-request reference audios", + file=sys.stderr, + ) + + elif voice_name and os.path.isfile(voice_name): ref_audio_path = self._get_ref_audio_path(voice_name) with open(ref_audio_path, "rb") as f: audio_bytes = f.read() diff --git a/core/http/endpoints/localai/tts.go b/core/http/endpoints/localai/tts.go index 16b49a91267f..f8d6c558fdc6 100644 --- a/core/http/endpoints/localai/tts.go +++ b/core/http/endpoints/localai/tts.go @@ -1,6 +1,7 @@ package localai import ( + "encoding/json" "errors" "fmt" "net/http" @@ -67,7 +68,7 @@ func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig if profiles == nil { return echo.NewHTTPError(http.StatusInternalServerError, "voice profile store is unavailable") } - profile, referencePath, release, err := profiles.LeaseAudio(c.Request().Context(), profileID) + profile, referencePaths, release, err := profiles.LeaseAudios(c.Request().Context(), profileID) if err != nil { if errors.Is(err, voiceprofile.ErrNotFound) { return echo.NewHTTPError(http.StatusNotFound, "voice profile not found") @@ -75,7 +76,7 @@ func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig return fmt.Errorf("resolve voice profile: %w", err) } defer release() - cfg.Voice = referencePath + cfg.Voice = referencePaths[0] if cfg.Language == "" && profile.Language != "" { cfg.Language = profile.Language } @@ -83,6 +84,20 @@ func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig input.Params = make(map[string]string) } input.Params["ref_text"] = profile.Transcript + if supportsMultipleVoiceReferences(cfg.Backend) && len(referencePaths) > 1 { + references := make([]map[string]string, 0, len(referencePaths)) + for index, path := range referencePaths { + references = append(references, map[string]string{"audio": path, "text": profile.References[index].Transcript}) + } + encoded, err := json.Marshal(references) + if err != nil { + return fmt.Errorf("encode voice profile references: %w", err) + } + input.Params["multi_reference_cond"] = string(encoded) + if cfg.Backend == "audio-cpp" { + cfg.Voice = "" + } + } xlog.Debug("Resolved saved voice profile", "id", profile.ID, "model", input.Model) } } @@ -138,3 +153,7 @@ func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig return c.Attachment(filePath, filepath.Base(filePath)) } } + +func supportsMultipleVoiceReferences(backendName string) bool { + return backendName == "fish-speech" || backendName == "audio-cpp" +} diff --git a/core/http/endpoints/localai/voice_profiles.go b/core/http/endpoints/localai/voice_profiles.go index 38990d212cb7..d2f19eb4a568 100644 --- a/core/http/endpoints/localai/voice_profiles.go +++ b/core/http/endpoints/localai/voice_profiles.go @@ -14,7 +14,7 @@ import ( "github.com/mudler/xlog" ) -const maxVoiceProfileJSONBytes = voiceprofile.MaxAudioBytes*4/3 + 2*1024*1024 +const maxVoiceProfileJSONBytes = voiceprofile.MaxAudioBytes*voiceprofile.MaxReferences*4/3 + 2*1024*1024 // VoiceProfileListResponse is returned by the profile library endpoint. type VoiceProfileListResponse struct { @@ -25,12 +25,18 @@ type VoiceProfileListResponse struct { // It is primarily used by the LocalAI admin MCP client; the browser sends a // multipart audio field to avoid base64 overhead. type CreateVoiceProfileRequest struct { - Name string `json:"name"` - Description string `json:"description,omitempty"` - Language string `json:"language,omitempty"` - Transcript string `json:"transcript"` - AudioBase64 string `json:"audio_base64"` - ConsentConfirmed bool `json:"consent_confirmed"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + Language string `json:"language,omitempty"` + Transcript string `json:"transcript"` + AudioBase64 string `json:"audio_base64"` + ConsentConfirmed bool `json:"consent_confirmed"` + References []CreateVoiceProfileReferenceRequest `json:"references,omitempty"` +} + +type CreateVoiceProfileReferenceRequest struct { + Transcript string `json:"transcript"` + AudioBase64 string `json:"audio_base64"` } func voiceProfileError(code int, message string) schema.ErrorResponse { @@ -118,31 +124,39 @@ func CreateVoiceProfileEndpoint(store *voiceprofile.Store) echo.HandlerFunc { contentType := c.Request().Header.Get(echo.HeaderContentType) if strings.HasPrefix(contentType, echo.MIMEMultipartForm) { - c.Request().Body = http.MaxBytesReader(c.Response().Writer, c.Request().Body, voiceprofile.MaxAudioBytes+2*1024*1024) - fileHeader, err := c.FormFile("audio") + c.Request().Body = http.MaxBytesReader(c.Response().Writer, c.Request().Body, voiceprofile.MaxAudioBytes*voiceprofile.MaxReferences+2*1024*1024) + form, err := c.MultipartForm() if err != nil { if isRequestBodyTooLarge(err) { return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge) } return writeVoiceProfileError(c, fmt.Errorf("%w: audio is required", voiceprofile.ErrInvalidInput)) } - if fileHeader.Size > voiceprofile.MaxAudioBytes { - return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge) + files := form.File["audio"] + transcripts := form.Value["transcript"] + if len(files) == 0 || len(files) != len(transcripts) || len(files) > voiceprofile.MaxReferences { + return writeVoiceProfileError(c, fmt.Errorf("%w: each audio needs one transcript", voiceprofile.ErrInvalidInput)) } - audio, err := fileHeader.Open() - if err != nil { - return writeVoiceProfileError(c, fmt.Errorf("open uploaded audio: %w", err)) + references := make([]voiceprofile.ReferenceInput, 0, len(files)) + for index, fileHeader := range files { + if fileHeader.Size > voiceprofile.MaxAudioBytes { + return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge) + } + audio, err := fileHeader.Open() + if err != nil { + return writeVoiceProfileError(c, fmt.Errorf("open uploaded audio: %w", err)) + } + defer func() { _ = audio.Close() }() + references = append(references, voiceprofile.ReferenceInput{Transcript: transcripts[index], Audio: audio}) } - defer func() { _ = audio.Close() }() consent, _ := strconv.ParseBool(c.FormValue("consent_confirmed")) - profile, err := store.Create(c.Request().Context(), voiceprofile.CreateInput{ + profile, err := store.CreateWithReferences(c.Request().Context(), voiceprofile.CreateInput{ Name: c.FormValue("name"), Description: c.FormValue("description"), Language: c.FormValue("language"), - Transcript: c.FormValue("transcript"), ConsentConfirmed: consent, - }, audio) + }, references) if err != nil { return writeVoiceProfileError(c, err) } @@ -157,17 +171,26 @@ func CreateVoiceProfileEndpoint(store *voiceprofile.Store) echo.HandlerFunc { } return writeVoiceProfileError(c, fmt.Errorf("%w: invalid JSON body", voiceprofile.ErrInvalidInput)) } - if base64.StdEncoding.DecodedLen(len(request.AudioBase64)) > int(voiceprofile.MaxAudioBytes) { - return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge) + referenceRequests := request.References + if len(referenceRequests) == 0 { + referenceRequests = []CreateVoiceProfileReferenceRequest{{Transcript: request.Transcript, AudioBase64: request.AudioBase64}} + } + if len(referenceRequests) > voiceprofile.MaxReferences { + return writeVoiceProfileError(c, voiceprofile.ErrInvalidInput) + } + references := make([]voiceprofile.ReferenceInput, 0, len(referenceRequests)) + for _, reference := range referenceRequests { + if base64.StdEncoding.DecodedLen(len(reference.AudioBase64)) > int(voiceprofile.MaxAudioBytes) { + return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge) + } + references = append(references, voiceprofile.ReferenceInput{Transcript: reference.Transcript, Audio: base64.NewDecoder(base64.StdEncoding, strings.NewReader(reference.AudioBase64))}) } - audio := base64.NewDecoder(base64.StdEncoding, strings.NewReader(request.AudioBase64)) - profile, err := store.Create(c.Request().Context(), voiceprofile.CreateInput{ + profile, err := store.CreateWithReferences(c.Request().Context(), voiceprofile.CreateInput{ Name: request.Name, Description: request.Description, Language: request.Language, - Transcript: request.Transcript, ConsentConfirmed: request.ConsentConfirmed, - }, audio) + }, references) if err != nil { return writeVoiceProfileError(c, err) } diff --git a/core/http/endpoints/localai/voice_profiles_test.go b/core/http/endpoints/localai/voice_profiles_test.go index c913f8355d37..e0425cb2141a 100644 --- a/core/http/endpoints/localai/voice_profiles_test.go +++ b/core/http/endpoints/localai/voice_profiles_test.go @@ -121,6 +121,26 @@ var _ = Describe("Voice profile endpoints", func() { Expect(recorder.Code).To(Equal(http.StatusCreated), recorder.Body.String()) }) + It("creates a profile with ordered JSON references", func() { + payload, err := json.Marshal(CreateVoiceProfileRequest{ + Name: "Personality", ConsentConfirmed: true, + References: []CreateVoiceProfileReferenceRequest{ + {Transcript: "First reference.", AudioBase64: base64.StdEncoding.EncodeToString(voiceProfileWAV(time.Second))}, + {Transcript: "Second reference.", AudioBase64: base64.StdEncoding.EncodeToString(voiceProfileWAV(2 * time.Second))}, + }, + }) + Expect(err).NotTo(HaveOccurred()) + request := httptest.NewRequest(http.MethodPost, "/api/voice-profiles", bytes.NewReader(payload)) + request.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + recorder := httptest.NewRecorder() + e.ServeHTTP(recorder, request) + Expect(recorder.Code).To(Equal(http.StatusCreated), recorder.Body.String()) + var created voiceprofile.Profile + Expect(json.Unmarshal(recorder.Body.Bytes(), &created)).To(Succeed()) + Expect(created.References).To(HaveLen(2)) + Expect(created.Transcript).To(Equal("First reference.")) + }) + It("rejects creation without explicit consent", func() { body, contentType := voiceProfileMultipart("false") request := httptest.NewRequest(http.MethodPost, "/api/voice-profiles", body) diff --git a/core/http/react-ui/e2e/voice-library.spec.js b/core/http/react-ui/e2e/voice-library.spec.js index 07b82ccb4920..0ef49f9e048d 100644 --- a/core/http/react-ui/e2e/voice-library.spec.js +++ b/core/http/react-ui/e2e/voice-library.spec.js @@ -60,7 +60,7 @@ async function mockVoiceAPIs(page) { return state } -test.describe('Voice Library', () => { +test.describe('Personality Library', () => { let apiState test.beforeEach(async ({ page }) => { @@ -69,7 +69,7 @@ test.describe('Voice Library', () => { test('renders the library-first master/detail view', async ({ page }) => { await page.goto('/app/voice-library') - await expect(page.getByRole('heading', { name: /Voice Library/i })).toBeVisible() + await expect(page.getByRole('heading', { name: /Personality Library/i })).toBeVisible() await expect(page.locator('.voice-row', { hasText: 'Documentary narrator' })).toBeVisible() await expect(page.locator('.voice-library-detail')).toContainText('The exact words spoken in this reference.') await expect(page.locator('.voice-library-detail')).toContainText('Consent confirmed') diff --git a/core/http/react-ui/public/locales/en/media.json b/core/http/react-ui/public/locales/en/media.json index 1b392d9d5c7b..af93d06f5b0c 100644 --- a/core/http/react-ui/public/locales/en/media.json +++ b/core/http/react-ui/public/locales/en/media.json @@ -172,25 +172,25 @@ }, "voiceLibrary": { "cloningReady": "Voice cloning", - "loading": "Loading saved voices…", + "loading": "Loading saved personalities…", "modelDefault": "Use model default", - "empty": "No saved voices yet.", + "empty": "No saved personalities yet.", "create": "Create one", - "manage": "Manage Voice Library", + "manage": "Manage Personality Library", "namedVoiceHint": "This model does not accept reference-audio profiles. You can still enter a named speaker supported by its backend." } }, "voiceLibrary": { - "title": "Voice Library", - "subtitle": "Create and manage reusable reference voices for every installed model that supports voice cloning.", - "loading": "Loading saved voices…", - "listLabel": "Saved voice profiles", + "title": "Personality Library", + "subtitle": "Create and manage reusable voice personalities with one or more reference recordings.", + "loading": "Loading saved personalities…", + "listLabel": "Saved voice personalities", "status": { "ready": "Ready" }, "summary": { "label": "Voice library status", - "profiles": "saved voices", + "profiles": "saved personalities", "modelsReady_one": "{{count}} compatible model ready", "modelsReady_other": "{{count}} compatible models ready", "noModels": "No compatible model installed" @@ -204,8 +204,8 @@ "allLanguages": "All languages" }, "empty": { - "title": "Your voice library is empty", - "body": "Record or upload one consented reference clip, add its exact transcript, and reuse it across compatible TTS models." + "title": "Your personality library is empty", + "body": "Record or upload consented reference clips, add their exact transcripts, and reuse the personality across compatible TTS models." }, "noResults": { "title": "No voices match these filters", @@ -251,12 +251,12 @@ "endpointNote": "The same model, input, and voice fields also work with POST /tts." }, "actions": { - "create": "Create voice", - "createFirst": "Create your first voice", + "create": "Create personality", + "createFirst": "Create your first personality", "retry": "Try again", "clearFilters": "Clear filters", "useInTTS": "Use in Text to Speech", - "delete": "Delete voice", + "delete": "Delete personality", "installModel": "Install a compatible model", "browseModels": "Browse all models" }, @@ -274,9 +274,14 @@ } }, "voiceCreate": { - "eyebrow": "Voice Library", - "title": "Create a reusable voice", - "subtitle": "Add one clean reference clip and its exact transcript. LocalAI handles the backend-specific cloning parameters at generation time.", + "eyebrow": "Personality Library", + "title": "Create a reusable personality", + "subtitle": "Create a reusable personality from one or more clean reference clips and their exact transcripts.", + "references": { + "add": "Add another reference", + "additional": "Reference {{number}}", + "remove": "Remove" + }, "sections": { "reference": { "title": "Reference audio", diff --git a/core/http/react-ui/src/pages/VoiceProfileCreate.jsx b/core/http/react-ui/src/pages/VoiceProfileCreate.jsx index 14d1caa02c66..502447ef9b66 100644 --- a/core/http/react-ui/src/pages/VoiceProfileCreate.jsx +++ b/core/http/react-ui/src/pages/VoiceProfileCreate.jsx @@ -66,6 +66,7 @@ export default function VoiceProfileCreate() { const [description, setDescription] = useState('') const [language, setLanguage] = useState('') const [transcript, setTranscript] = useState('') + const [additionalReferences, setAdditionalReferences] = useState([]) const [consent, setConsent] = useState(false) const [submitting, setSubmitting] = useState(false) @@ -75,7 +76,8 @@ export default function VoiceProfileCreate() { const durationValid = audio?.duration >= 1 && audio?.duration <= 120 const durationRecommended = audio?.duration >= 6 && audio?.duration <= 30 - const formReady = !!audio && durationValid && name.trim() && transcript.trim() && consent && !audioProcessing + const additionalReady = additionalReferences.every(reference => reference.audio && reference.transcript.trim()) + const formReady = !!audio && durationValid && name.trim() && transcript.trim() && additionalReady && consent && !audioProcessing const readiness = useMemo(() => ({ audio: !!audio && durationValid, @@ -108,6 +110,23 @@ export default function VoiceProfileCreate() { } } + const handleAdditionalAudio = async (index, sample) => { + if (!sample) { + setAdditionalReferences(current => current.map((reference, itemIndex) => itemIndex === index ? { ...reference, audio: null } : reference)) + return + } + setAudioProcessing(true) + try { + const normalized = await normalizeAudioSample(sample) + if (normalized.duration < 1 || normalized.duration > 120) throw new Error(t('voiceCreate.audio.durationError')) + setAdditionalReferences(current => current.map((reference, itemIndex) => itemIndex === index ? { ...reference, audio: normalized } : reference)) + } catch (err) { + addToast(err.message || t('voiceCreate.audio.decodeError'), 'error') + } finally { + setAudioProcessing(false) + } + } + const submit = async (event) => { event.preventDefault() if (!formReady) return @@ -120,6 +139,10 @@ export default function VoiceProfileCreate() { formData.append('transcript', transcript.trim()) formData.append('consent_confirmed', 'true') formData.append('audio', audio.blob, 'reference.wav') + additionalReferences.forEach((reference, index) => { + formData.append('transcript', reference.transcript.trim()) + formData.append('audio', reference.audio.blob, `reference-${index + 2}.wav`) + }) const profile = await voiceProfilesApi.create(formData) addToast(t('voiceCreate.toasts.created', { name: profile.name }), 'success') navigate(`/app/voice-library?selected=${encodeURIComponent(profile.id)}`) @@ -133,7 +156,7 @@ export default function VoiceProfileCreate() { return (
)} + {additionalReferences.map((reference, index) => ( +
+
+ {t('voiceCreate.references.additional', { number: index + 2 })} + +
+ handleAdditionalAudio(index, sample)} maxBytes={MAX_AUDIO_BYTES} preferBlob idPrefix={`voice-profile-${index + 2}`} /> + +