feat(tts): support multi-reference personalities

backend/cpp/audio-cpp/generation_request.cpp

@@ -22,7 +22,8 @@ bool voice_is_reference_file(const std::string &voice) {
 
 RequestShape build_tts_shape(const backend::TTSRequest &request) {
     RequestShape shape;
-    shape.has_voice_reference = voice_is_reference_file(request.voice());
+    shape.has_voice_reference = voice_is_reference_file(request.voice()) ||
+                                request.params().find("multi_reference_cond") != request.params().end();
     // !empty() as well as has_instructions(), and it must match the guard in
     // build_tts_request: a request whose instructions are an empty string
     // carries no style condition, so telling routing to prefer VoiceDesign for

backend/cpp/audio-cpp/generation_request_ctest.cpp

@@ -128,6 +128,14 @@ static void test_tts_shape() {
         check(!shape.has_voice_reference,
               "shape: a preset name is not a voice reference");
     }
+    {
+        backend::TTSRequest request;
+        (*request.mutable_params())["multi_reference_cond"] =
+            R"([{"audio":"one.wav","text":"one"}])";
+        const auto shape = build_tts_shape(request);
+        check(shape.has_voice_reference,
+              "shape: multi-reference conditioning is a voice reference");
+    }
     {
         backend::TTSRequest request;
         request.set_voice(dir.string());

backend/python/fish-speech/backend.py

@@ -326,11 +326,29 @@ def TTS(self, request, context):
             max_new_tokens = self.options.get("max_new_tokens", 1024)
             chunk_length = self.options.get("chunk_length", 200)
 
-            # Build references list for voice cloning
+            # Build references list for voice cloning. Saved LocalAI
+            # personalities use the same ordered JSON shape as audio.cpp.
             references = []
             voice_name = request.voice if request.voice else None
 
-            if voice_name and os.path.isfile(voice_name):
+            multi_reference_cond = request.params.get("multi_reference_cond", "") if hasattr(request, "params") else ""
+            if multi_reference_cond:
+                reference_entries = json.loads(multi_reference_cond)
+                if not isinstance(reference_entries, list) or not reference_entries:
+                    raise ValueError("multi_reference_cond must be a non-empty JSON array")
+                for entry in reference_entries:
+                    if not isinstance(entry, dict) or not entry.get("audio") or not entry.get("text"):
+                        raise ValueError("multi_reference_cond entries require audio and text")
+                    ref_audio_path = self._get_ref_audio_path(entry["audio"])
+                    with open(ref_audio_path, "rb") as f:
+                        audio_bytes = f.read()
+                    references.append(ServeReferenceAudio(audio=audio_bytes, text=entry["text"]))
+                print(
+                    f"[INFO] Using {len(references)} per-request reference audios",
+                    file=sys.stderr,
+                )
+
+            elif voice_name and os.path.isfile(voice_name):
                 ref_audio_path = self._get_ref_audio_path(voice_name)
                 with open(ref_audio_path, "rb") as f:
                     audio_bytes = f.read()

core/http/endpoints/localai/tts.go

@@ -1,6 +1,7 @@
 package localai
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
@@ -67,22 +68,36 @@ func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig
 				if profiles == nil {
 					return echo.NewHTTPError(http.StatusInternalServerError, "voice profile store is unavailable")
 				}
-				profile, referencePath, release, err := profiles.LeaseAudio(c.Request().Context(), profileID)
+				profile, referencePaths, release, err := profiles.LeaseAudios(c.Request().Context(), profileID)
 				if err != nil {
 					if errors.Is(err, voiceprofile.ErrNotFound) {
 						return echo.NewHTTPError(http.StatusNotFound, "voice profile not found")
 					}
 					return fmt.Errorf("resolve voice profile: %w", err)
 				}
 				defer release()
-				cfg.Voice = referencePath
+				cfg.Voice = referencePaths[0]
 				if cfg.Language == "" && profile.Language != "" {
 					cfg.Language = profile.Language
 				}
 				if input.Params == nil {
 					input.Params = make(map[string]string)
 				}
 				input.Params["ref_text"] = profile.Transcript
+				if supportsMultipleVoiceReferences(cfg.Backend) && len(referencePaths) > 1 {
+					references := make([]map[string]string, 0, len(referencePaths))
+					for index, path := range referencePaths {
+						references = append(references, map[string]string{"audio": path, "text": profile.References[index].Transcript})
+					}
+					encoded, err := json.Marshal(references)
+					if err != nil {
+						return fmt.Errorf("encode voice profile references: %w", err)
+					}
+					input.Params["multi_reference_cond"] = string(encoded)
+					if cfg.Backend == "audio-cpp" {
+						cfg.Voice = ""
+					}
+				}
 				xlog.Debug("Resolved saved voice profile", "id", profile.ID, "model", input.Model)
 			}
 		}
@@ -138,3 +153,7 @@ func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig
 		return c.Attachment(filePath, filepath.Base(filePath))
 	}
 }
+
+func supportsMultipleVoiceReferences(backendName string) bool {
+	return backendName == "fish-speech" || backendName == "audio-cpp"
+}

core/http/endpoints/localai/voice_profiles.go

@@ -14,7 +14,7 @@ import (
 	"github.com/mudler/xlog"
 )
 
-const maxVoiceProfileJSONBytes = voiceprofile.MaxAudioBytes*4/3 + 2*1024*1024
+const maxVoiceProfileJSONBytes = voiceprofile.MaxAudioBytes*voiceprofile.MaxReferences*4/3 + 2*1024*1024
 
 // VoiceProfileListResponse is returned by the profile library endpoint.
 type VoiceProfileListResponse struct {
@@ -25,12 +25,18 @@ type VoiceProfileListResponse struct {
 // It is primarily used by the LocalAI admin MCP client; the browser sends a
 // multipart audio field to avoid base64 overhead.
 type CreateVoiceProfileRequest struct {
-	Name             string `json:"name"`
-	Description      string `json:"description,omitempty"`
-	Language         string `json:"language,omitempty"`
-	Transcript       string `json:"transcript"`
-	AudioBase64      string `json:"audio_base64"`
-	ConsentConfirmed bool   `json:"consent_confirmed"`
+	Name             string                               `json:"name"`
+	Description      string                               `json:"description,omitempty"`
+	Language         string                               `json:"language,omitempty"`
+	Transcript       string                               `json:"transcript"`
+	AudioBase64      string                               `json:"audio_base64"`
+	ConsentConfirmed bool                                 `json:"consent_confirmed"`
+	References       []CreateVoiceProfileReferenceRequest `json:"references,omitempty"`
+}
+
+type CreateVoiceProfileReferenceRequest struct {
+	Transcript  string `json:"transcript"`
+	AudioBase64 string `json:"audio_base64"`
 }
 
 func voiceProfileError(code int, message string) schema.ErrorResponse {
@@ -118,31 +124,39 @@ func CreateVoiceProfileEndpoint(store *voiceprofile.Store) echo.HandlerFunc {
 
 		contentType := c.Request().Header.Get(echo.HeaderContentType)
 		if strings.HasPrefix(contentType, echo.MIMEMultipartForm) {
-			c.Request().Body = http.MaxBytesReader(c.Response().Writer, c.Request().Body, voiceprofile.MaxAudioBytes+2*1024*1024)
-			fileHeader, err := c.FormFile("audio")
+			c.Request().Body = http.MaxBytesReader(c.Response().Writer, c.Request().Body, voiceprofile.MaxAudioBytes*voiceprofile.MaxReferences+2*1024*1024)
+			form, err := c.MultipartForm()
 			if err != nil {
 				if isRequestBodyTooLarge(err) {
 					return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge)
 				}
 				return writeVoiceProfileError(c, fmt.Errorf("%w: audio is required", voiceprofile.ErrInvalidInput))
 			}
-			if fileHeader.Size > voiceprofile.MaxAudioBytes {
-				return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge)
+			files := form.File["audio"]
+			transcripts := form.Value["transcript"]
+			if len(files) == 0 || len(files) != len(transcripts) || len(files) > voiceprofile.MaxReferences {
+				return writeVoiceProfileError(c, fmt.Errorf("%w: each audio needs one transcript", voiceprofile.ErrInvalidInput))
 			}
-			audio, err := fileHeader.Open()
-			if err != nil {
-				return writeVoiceProfileError(c, fmt.Errorf("open uploaded audio: %w", err))
+			references := make([]voiceprofile.ReferenceInput, 0, len(files))
+			for index, fileHeader := range files {
+				if fileHeader.Size > voiceprofile.MaxAudioBytes {
+					return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge)
+				}
+				audio, err := fileHeader.Open()
+				if err != nil {
+					return writeVoiceProfileError(c, fmt.Errorf("open uploaded audio: %w", err))
+				}
+				defer func() { _ = audio.Close() }()
+				references = append(references, voiceprofile.ReferenceInput{Transcript: transcripts[index], Audio: audio})
 			}
-			defer func() { _ = audio.Close() }()
 
 			consent, _ := strconv.ParseBool(c.FormValue("consent_confirmed"))
-			profile, err := store.Create(c.Request().Context(), voiceprofile.CreateInput{
+			profile, err := store.CreateWithReferences(c.Request().Context(), voiceprofile.CreateInput{
 				Name:             c.FormValue("name"),
 				Description:      c.FormValue("description"),
 				Language:         c.FormValue("language"),
-				Transcript:       c.FormValue("transcript"),
 				ConsentConfirmed: consent,
-			}, audio)
+			}, references)
 			if err != nil {
 				return writeVoiceProfileError(c, err)
 			}
@@ -157,17 +171,26 @@ func CreateVoiceProfileEndpoint(store *voiceprofile.Store) echo.HandlerFunc {
 			}
 			return writeVoiceProfileError(c, fmt.Errorf("%w: invalid JSON body", voiceprofile.ErrInvalidInput))
 		}
-		if base64.StdEncoding.DecodedLen(len(request.AudioBase64)) > int(voiceprofile.MaxAudioBytes) {
-			return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge)
+		referenceRequests := request.References
+		if len(referenceRequests) == 0 {
+			referenceRequests = []CreateVoiceProfileReferenceRequest{{Transcript: request.Transcript, AudioBase64: request.AudioBase64}}
+		}
+		if len(referenceRequests) > voiceprofile.MaxReferences {
+			return writeVoiceProfileError(c, voiceprofile.ErrInvalidInput)
+		}
+		references := make([]voiceprofile.ReferenceInput, 0, len(referenceRequests))
+		for _, reference := range referenceRequests {
+			if base64.StdEncoding.DecodedLen(len(reference.AudioBase64)) > int(voiceprofile.MaxAudioBytes) {
+				return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge)
+			}
+			references = append(references, voiceprofile.ReferenceInput{Transcript: reference.Transcript, Audio: base64.NewDecoder(base64.StdEncoding, strings.NewReader(reference.AudioBase64))})
 		}
-		audio := base64.NewDecoder(base64.StdEncoding, strings.NewReader(request.AudioBase64))
-		profile, err := store.Create(c.Request().Context(), voiceprofile.CreateInput{
+		profile, err := store.CreateWithReferences(c.Request().Context(), voiceprofile.CreateInput{
 			Name:             request.Name,
 			Description:      request.Description,
 			Language:         request.Language,
-			Transcript:       request.Transcript,
 			ConsentConfirmed: request.ConsentConfirmed,
-		}, audio)
+		}, references)
 		if err != nil {
 			return writeVoiceProfileError(c, err)
 		}

core/http/endpoints/localai/voice_profiles_test.go

@@ -121,6 +121,26 @@ var _ = Describe("Voice profile endpoints", func() {
 		Expect(recorder.Code).To(Equal(http.StatusCreated), recorder.Body.String())
 	})
 
+	It("creates a profile with ordered JSON references", func() {
+		payload, err := json.Marshal(CreateVoiceProfileRequest{
+			Name: "Personality", ConsentConfirmed: true,
+			References: []CreateVoiceProfileReferenceRequest{
+				{Transcript: "First reference.", AudioBase64: base64.StdEncoding.EncodeToString(voiceProfileWAV(time.Second))},
+				{Transcript: "Second reference.", AudioBase64: base64.StdEncoding.EncodeToString(voiceProfileWAV(2 * time.Second))},
+			},
+		})
+		Expect(err).NotTo(HaveOccurred())
+		request := httptest.NewRequest(http.MethodPost, "/api/voice-profiles", bytes.NewReader(payload))
+		request.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+		recorder := httptest.NewRecorder()
+		e.ServeHTTP(recorder, request)
+		Expect(recorder.Code).To(Equal(http.StatusCreated), recorder.Body.String())
+		var created voiceprofile.Profile
+		Expect(json.Unmarshal(recorder.Body.Bytes(), &created)).To(Succeed())
+		Expect(created.References).To(HaveLen(2))
+		Expect(created.Transcript).To(Equal("First reference."))
+	})
+
 	It("rejects creation without explicit consent", func() {
 		body, contentType := voiceProfileMultipart("false")
 		request := httptest.NewRequest(http.MethodPost, "/api/voice-profiles", body)

core/http/react-ui/e2e/voice-library.spec.js

@@ -60,7 +60,7 @@ async function mockVoiceAPIs(page) {
   return state
 }
 
-test.describe('Voice Library', () => {
+test.describe('Personality Library', () => {
   let apiState
 
   test.beforeEach(async ({ page }) => {
@@ -69,7 +69,7 @@ test.describe('Voice Library', () => {
 
   test('renders the library-first master/detail view', async ({ page }) => {
     await page.goto('/app/voice-library')
-    await expect(page.getByRole('heading', { name: /Voice Library/i })).toBeVisible()
+    await expect(page.getByRole('heading', { name: /Personality Library/i })).toBeVisible()
     await expect(page.locator('.voice-row', { hasText: 'Documentary narrator' })).toBeVisible()
     await expect(page.locator('.voice-library-detail')).toContainText('The exact words spoken in this reference.')
     await expect(page.locator('.voice-library-detail')).toContainText('Consent confirmed')

core/http/react-ui/public/locales/en/media.json

@@ -172,25 +172,25 @@
     },
     "voiceLibrary": {
       "cloningReady": "Voice cloning",
-      "loading": "Loading saved voices…",
+      "loading": "Loading saved personalities…",
       "modelDefault": "Use model default",
-      "empty": "No saved voices yet.",
+      "empty": "No saved personalities yet.",
       "create": "Create one",
-      "manage": "Manage Voice Library",
+      "manage": "Manage Personality Library",
       "namedVoiceHint": "This model does not accept reference-audio profiles. You can still enter a named speaker supported by its backend."
     }
   },
   "voiceLibrary": {
-    "title": "Voice Library",
-    "subtitle": "Create and manage reusable reference voices for every installed model that supports voice cloning.",
-    "loading": "Loading saved voices…",
-    "listLabel": "Saved voice profiles",
+    "title": "Personality Library",
+    "subtitle": "Create and manage reusable voice personalities with one or more reference recordings.",
+    "loading": "Loading saved personalities…",
+    "listLabel": "Saved voice personalities",
     "status": {
       "ready": "Ready"
     },
     "summary": {
       "label": "Voice library status",
-      "profiles": "saved voices",
+      "profiles": "saved personalities",
       "modelsReady_one": "{{count}} compatible model ready",
       "modelsReady_other": "{{count}} compatible models ready",
       "noModels": "No compatible model installed"
@@ -204,8 +204,8 @@
       "allLanguages": "All languages"
     },
     "empty": {
-      "title": "Your voice library is empty",
-      "body": "Record or upload one consented reference clip, add its exact transcript, and reuse it across compatible TTS models."
+      "title": "Your personality library is empty",
+      "body": "Record or upload consented reference clips, add their exact transcripts, and reuse the personality across compatible TTS models."
     },
     "noResults": {
       "title": "No voices match these filters",
@@ -251,12 +251,12 @@
       "endpointNote": "The same model, input, and voice fields also work with POST /tts."
     },
     "actions": {
-      "create": "Create voice",
-      "createFirst": "Create your first voice",
+      "create": "Create personality",
+      "createFirst": "Create your first personality",
       "retry": "Try again",
       "clearFilters": "Clear filters",
       "useInTTS": "Use in Text to Speech",
-      "delete": "Delete voice",
+      "delete": "Delete personality",
       "installModel": "Install a compatible model",
       "browseModels": "Browse all models"
     },
@@ -274,9 +274,14 @@
     }
   },
   "voiceCreate": {
-    "eyebrow": "Voice Library",
-    "title": "Create a reusable voice",
-    "subtitle": "Add one clean reference clip and its exact transcript. LocalAI handles the backend-specific cloning parameters at generation time.",
+    "eyebrow": "Personality Library",
+    "title": "Create a reusable personality",
+    "subtitle": "Create a reusable personality from one or more clean reference clips and their exact transcripts.",
+    "references": {
+      "add": "Add another reference",
+      "additional": "Reference {{number}}",
+      "remove": "Remove"
+    },
     "sections": {
       "reference": {
         "title": "Reference audio",