From 1b222045daf976ec7dd36fdece2d019d2e34e3a9 Mon Sep 17 00:00:00 2001 From: s546126 Date: Sun, 9 Aug 2026 16:23:09 -0700 Subject: [PATCH] Add WebRTC realtime voice provider (OpenAI Realtime) on Android Extract GeminiLiveService's public surface into a provider-neutral RealtimeVoiceService interface and add an OpenAI Realtime backend that runs over WebRTC: Opus audio with built-in echo cancellation and jitter buffering, model/tool events over the oai-events data channel, and the documented ephemeral-key flow (client_secrets + SDP exchange via /v1/realtime/calls). Providers that manage their own audio (WebRTC) bypass the PCM AudioManager pump entirely, which also sidesteps the known Gemini-Live vs WebRTC audio device conflict on that path. Vision is per-turn for OpenAI: the latest camera frame is attached as an image item when the server detects speech start, instead of a continuous 1fps stream. Tool responses are now built inside each provider (Gemini toolResponse JSON vs OpenAI function_call_output), so ToolCallRouter hands back (callId, name, result) instead of pre-shaped Gemini JSON. Co-Authored-By: Claude Fable 5 --- README.md | 11 + .../cameraaccess/gemini/GeminiLiveService.kt | 88 +-- .../gemini/GeminiSessionViewModel.kt | 110 ++-- .../cameraaccess/openclaw/ToolCallModels.kt | 36 +- .../cameraaccess/openclaw/ToolCallRouter.kt | 25 +- .../cameraaccess/settings/SettingsManager.kt | 9 + .../cameraaccess/ui/GeminiOverlayView.kt | 14 +- .../cameraaccess/ui/SettingsScreen.kt | 33 ++ .../voice/RealtimeVoiceService.kt | 60 +++ .../voice/VoiceConnectionState.kt | 11 + .../voice/openai/OpenAIRealtimeConfig.kt | 27 + .../voice/openai/OpenAIRealtimeService.kt | 502 ++++++++++++++++++ 12 files changed, 803 insertions(+), 123 deletions(-) create mode 100644 samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/voice/RealtimeVoiceService.kt create mode 100644 samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/voice/VoiceConnectionState.kt create mode 100644 samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/voice/openai/OpenAIRealtimeConfig.kt create mode 100644 samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/voice/openai/OpenAIRealtimeService.kt diff --git a/README.md b/README.md index 1e66a6492..112cc1e46 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ Gemini Live API (WebSocket) **Key pieces:** - **Gemini Live** -- real-time voice + vision AI over WebSocket (native audio, not STT-first) +- **OpenAI Realtime (Android)** -- optional alternate voice backend over WebRTC (Opus, built-in AEC, full-duplex barge-in); switch providers in Settings - **OpenClaw** (optional) -- local gateway that gives Gemini access to 56+ tools and all your connected apps - **Phone mode** -- test the full pipeline using your phone camera instead of glasses - **WebRTC streaming** -- share your glasses POV live to a browser viewer @@ -262,6 +263,8 @@ All source code is in `samples/CameraAccessAndroid/app/src/main/java/.../cameraa | `gemini/GeminiLiveService.kt` | OkHttp WebSocket client for Gemini Live API | | `gemini/AudioManager.kt` | AudioRecord (16kHz) + AudioTrack (24kHz) | | `gemini/GeminiSessionViewModel.kt` | Session lifecycle, tool call wiring, UI state | +| `voice/RealtimeVoiceService.kt` | Provider-neutral voice session interface + factory | +| `voice/openai/OpenAIRealtimeService.kt` | OpenAI Realtime API over WebRTC (audio + data channel) | | `openclaw/ToolCallModels.kt` | Tool declarations, data classes | | `openclaw/OpenClawBridge.kt` | OkHttp HTTP client for OpenClaw gateway | | `openclaw/ToolCallRouter.kt` | Routes Gemini tool calls to OpenClaw | @@ -277,6 +280,14 @@ All source code is in `samples/CameraAccessAndroid/app/src/main/java/.../cameraa - **iOS iPhone mode**: Uses `.voiceChat` audio session for echo cancellation + mic gating during AI speech - **iOS Glasses mode**: Uses `.videoChat` audio session (mic is on glasses, speaker is on phone -- no echo) - **Android**: Uses `VOICE_COMMUNICATION` audio source for built-in acoustic echo cancellation +- **Android, OpenAI Realtime provider**: WebRTC owns mic and speaker end to end (Opus, jitter buffer, echo cancellation inside the peer connection); the PCM pipeline above stays idle + +#### Voice Providers (Android) + +Settings -> Voice Provider switches between: + +- **Gemini Live** (default) -- WebSocket, PCM audio pumped by `AudioManager`, continuous ~1fps vision frames +- **OpenAI Realtime** -- WebRTC via the ephemeral-key flow (`/v1/realtime/client_secrets` then SDP exchange with `/v1/realtime/calls`); model/tool events flow over the `oai-events` data channel. Vision is per-turn: the latest camera frame is attached as an image item when you start speaking, instead of a continuous stream. Requires an OpenAI API key in Settings. ### Video Pipeline diff --git a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiLiveService.kt b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiLiveService.kt index d046d306f..d77cc767a 100644 --- a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiLiveService.kt +++ b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiLiveService.kt @@ -6,6 +6,9 @@ import android.util.Log import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.GeminiToolCall import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.GeminiToolCallCancellation import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.ToolDeclarations +import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.ToolResult +import com.meta.wearable.dat.externalsampleapps.cameraaccess.voice.RealtimeVoiceService +import com.meta.wearable.dat.externalsampleapps.cameraaccess.voice.VoiceConnectionState import java.io.ByteArrayOutputStream import java.util.Timer import java.util.TimerTask @@ -23,33 +26,27 @@ import okio.ByteString import org.json.JSONArray import org.json.JSONObject -sealed class GeminiConnectionState { - data object Disconnected : GeminiConnectionState() - data object Connecting : GeminiConnectionState() - data object SettingUp : GeminiConnectionState() - data object Ready : GeminiConnectionState() - data class Error(val message: String) : GeminiConnectionState() -} - -class GeminiLiveService { +class GeminiLiveService : RealtimeVoiceService { companion object { private const val TAG = "GeminiLiveService" } - private val _connectionState = MutableStateFlow(GeminiConnectionState.Disconnected) - val connectionState: StateFlow = _connectionState.asStateFlow() + private val _connectionState = MutableStateFlow(VoiceConnectionState.Disconnected) + override val connectionState: StateFlow = _connectionState.asStateFlow() private val _isModelSpeaking = MutableStateFlow(false) - val isModelSpeaking: StateFlow = _isModelSpeaking.asStateFlow() + override val isModelSpeaking: StateFlow = _isModelSpeaking.asStateFlow() + + override val managesOwnAudio: Boolean = false - var onAudioReceived: ((ByteArray) -> Unit)? = null - var onTurnComplete: (() -> Unit)? = null - var onInterrupted: (() -> Unit)? = null - var onDisconnected: ((String?) -> Unit)? = null - var onInputTranscription: ((String) -> Unit)? = null - var onOutputTranscription: ((String) -> Unit)? = null - var onToolCall: ((GeminiToolCall) -> Unit)? = null - var onToolCallCancellation: ((GeminiToolCallCancellation) -> Unit)? = null + override var onAudioReceived: ((ByteArray) -> Unit)? = null + override var onTurnComplete: (() -> Unit)? = null + override var onInterrupted: (() -> Unit)? = null + override var onDisconnected: ((String?) -> Unit)? = null + override var onInputTranscription: ((String) -> Unit)? = null + override var onOutputTranscription: ((String) -> Unit)? = null + override var onToolCall: ((GeminiToolCall) -> Unit)? = null + override var onToolCallCancellation: ((GeminiToolCallCancellation) -> Unit)? = null // Latency tracking private var lastUserSpeechEnd: Long = 0 @@ -65,22 +62,22 @@ class GeminiLiveService { .pingInterval(10, TimeUnit.SECONDS) .build() - fun connect(callback: (Boolean) -> Unit) { + override fun connect(callback: (Boolean) -> Unit) { val url = GeminiConfig.websocketURL() if (url == null) { - _connectionState.value = GeminiConnectionState.Error("No API key configured") + _connectionState.value = VoiceConnectionState.Error("No API key configured") callback(false) return } - _connectionState.value = GeminiConnectionState.Connecting + _connectionState.value = VoiceConnectionState.Connecting connectCallback = callback val request = Request.Builder().url(url).build() webSocket = client.newWebSocket(request, object : WebSocketListener() { override fun onOpen(webSocket: WebSocket, response: Response) { Log.d(TAG, "WebSocket opened") - _connectionState.value = GeminiConnectionState.SettingUp + _connectionState.value = VoiceConnectionState.SettingUp sendSetupMessage() } @@ -95,7 +92,7 @@ class GeminiLiveService { override fun onFailure(webSocket: WebSocket, t: Throwable, response: Response?) { val msg = t.message ?: "Unknown error" Log.e(TAG, "WebSocket failure: $msg") - _connectionState.value = GeminiConnectionState.Error(msg) + _connectionState.value = VoiceConnectionState.Error(msg) _isModelSpeaking.value = false resolveConnect(false) onDisconnected?.invoke(msg) @@ -103,7 +100,7 @@ class GeminiLiveService { override fun onClosing(webSocket: WebSocket, code: Int, reason: String) { Log.d(TAG, "WebSocket closing: $code $reason") - _connectionState.value = GeminiConnectionState.Disconnected + _connectionState.value = VoiceConnectionState.Disconnected _isModelSpeaking.value = false resolveConnect(false) onDisconnected?.invoke("Connection closed (code $code: $reason)") @@ -111,7 +108,7 @@ class GeminiLiveService { override fun onClosed(webSocket: WebSocket, code: Int, reason: String) { Log.d(TAG, "WebSocket closed: $code $reason") - _connectionState.value = GeminiConnectionState.Disconnected + _connectionState.value = VoiceConnectionState.Disconnected _isModelSpeaking.value = false } }) @@ -120,10 +117,10 @@ class GeminiLiveService { timeoutTimer = Timer().apply { schedule(object : TimerTask() { override fun run() { - if (_connectionState.value == GeminiConnectionState.Connecting - || _connectionState.value == GeminiConnectionState.SettingUp) { + if (_connectionState.value == VoiceConnectionState.Connecting + || _connectionState.value == VoiceConnectionState.SettingUp) { Log.e(TAG, "Connection timed out") - _connectionState.value = GeminiConnectionState.Error("Connection timed out") + _connectionState.value = VoiceConnectionState.Error("Connection timed out") resolveConnect(false) } } @@ -131,20 +128,20 @@ class GeminiLiveService { } } - fun disconnect() { + override fun disconnect() { timeoutTimer?.cancel() timeoutTimer = null webSocket?.close(1000, null) webSocket = null onToolCall = null onToolCallCancellation = null - _connectionState.value = GeminiConnectionState.Disconnected + _connectionState.value = VoiceConnectionState.Disconnected _isModelSpeaking.value = false resolveConnect(false) } - fun sendAudio(data: ByteArray) { - if (_connectionState.value != GeminiConnectionState.Ready) return + override fun sendAudio(data: ByteArray) { + if (_connectionState.value != VoiceConnectionState.Ready) return sendExecutor.execute { val base64 = Base64.encodeToString(data, Base64.NO_WRAP) val json = JSONObject().apply { @@ -159,8 +156,8 @@ class GeminiLiveService { } } - fun sendVideoFrame(bitmap: Bitmap) { - if (_connectionState.value != GeminiConnectionState.Ready) return + override fun sendVideoFrame(bitmap: Bitmap) { + if (_connectionState.value != VoiceConnectionState.Ready) return sendExecutor.execute { val baos = ByteArrayOutputStream() bitmap.compress(Bitmap.CompressFormat.JPEG, GeminiConfig.VIDEO_JPEG_QUALITY, baos) @@ -177,14 +174,23 @@ class GeminiLiveService { } } - fun sendToolResponse(response: JSONObject) { + override fun sendToolResponse(callId: String, name: String, result: ToolResult) { + val response = JSONObject().apply { + put("toolResponse", JSONObject().apply { + put("functionResponses", JSONArray().put(JSONObject().apply { + put("id", callId) + put("name", name) + put("response", result.toJSON()) + })) + }) + } sendExecutor.execute { webSocket?.send(response.toString()) } } - fun sendTextMessage(text: String) { - if (_connectionState.value != GeminiConnectionState.Ready) return + override fun sendTextMessage(text: String) { + if (_connectionState.value != VoiceConnectionState.Ready) return sendExecutor.execute { val json = JSONObject().apply { put("clientContent", JSONObject().apply { @@ -258,7 +264,7 @@ class GeminiLiveService { // Setup complete if (json.has("setupComplete")) { - _connectionState.value = GeminiConnectionState.Ready + _connectionState.value = VoiceConnectionState.Ready resolveConnect(true) return } @@ -267,7 +273,7 @@ class GeminiLiveService { if (json.has("goAway")) { val goAway = json.getJSONObject("goAway") val seconds = goAway.optJSONObject("timeLeft")?.optInt("seconds", 0) ?: 0 - _connectionState.value = GeminiConnectionState.Disconnected + _connectionState.value = VoiceConnectionState.Disconnected _isModelSpeaking.value = false onDisconnected?.invoke("Server closing (time left: ${seconds}s)") return diff --git a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiSessionViewModel.kt b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiSessionViewModel.kt index 31567442a..90da1e739 100644 --- a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiSessionViewModel.kt +++ b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/gemini/GeminiSessionViewModel.kt @@ -1,8 +1,9 @@ package com.meta.wearable.dat.externalsampleapps.cameraaccess.gemini +import android.app.Application import android.graphics.Bitmap import android.util.Log -import androidx.lifecycle.ViewModel +import androidx.lifecycle.AndroidViewModel import androidx.lifecycle.viewModelScope import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.OpenClawBridge import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.OpenClawEventClient @@ -11,6 +12,11 @@ import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.OpenClawCo import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.ToolCallRouter import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.ToolCallStatus import com.meta.wearable.dat.externalsampleapps.cameraaccess.stream.StreamingMode +import com.meta.wearable.dat.externalsampleapps.cameraaccess.voice.RealtimeVoiceService +import com.meta.wearable.dat.externalsampleapps.cameraaccess.voice.VoiceConnectionState +import com.meta.wearable.dat.externalsampleapps.cameraaccess.voice.VoiceProvider +import com.meta.wearable.dat.externalsampleapps.cameraaccess.voice.VoiceServiceFactory +import com.meta.wearable.dat.externalsampleapps.cameraaccess.voice.openai.OpenAIRealtimeConfig import kotlinx.coroutines.Job import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow @@ -21,7 +27,7 @@ import kotlinx.coroutines.launch data class GeminiUiState( val isGeminiActive: Boolean = false, - val connectionState: GeminiConnectionState = GeminiConnectionState.Disconnected, + val connectionState: VoiceConnectionState = VoiceConnectionState.Disconnected, val isModelSpeaking: Boolean = false, val errorMessage: String? = null, val userTranscript: String = "", @@ -30,7 +36,7 @@ data class GeminiUiState( val openClawConnectionState: OpenClawConnectionState = OpenClawConnectionState.NotConfigured, ) -class GeminiSessionViewModel : ViewModel() { +class GeminiSessionViewModel(application: Application) : AndroidViewModel(application) { companion object { private const val TAG = "GeminiSessionVM" } @@ -38,7 +44,10 @@ class GeminiSessionViewModel : ViewModel() { private val _uiState = MutableStateFlow(GeminiUiState()) val uiState: StateFlow = _uiState.asStateFlow() - private val geminiService = GeminiLiveService() + private val application: Application = application + // Re-created on every startSession() so a provider switch in Settings + // takes effect on the next session. + private var voiceService: RealtimeVoiceService = VoiceServiceFactory.create(application) private val openClawBridge = OpenClawBridge() private var toolCallRouter: ToolCallRouter? = null private val audioManager = AudioManager() @@ -51,48 +60,61 @@ class GeminiSessionViewModel : ViewModel() { fun startSession() { if (_uiState.value.isGeminiActive) return - if (!GeminiConfig.isConfigured) { - _uiState.value = _uiState.value.copy( - errorMessage = "Gemini API key not configured. Open Settings and add your key from https://aistudio.google.com/apikey" - ) - return + when (VoiceServiceFactory.current()) { + VoiceProvider.GEMINI -> if (!GeminiConfig.isConfigured) { + _uiState.value = _uiState.value.copy( + errorMessage = "Gemini API key not configured. Open Settings and add your key from https://aistudio.google.com/apikey" + ) + return + } + VoiceProvider.OPENAI_REALTIME -> if (!OpenAIRealtimeConfig.isConfigured) { + _uiState.value = _uiState.value.copy( + errorMessage = "OpenAI API key not configured. Open Settings and add your key from https://platform.openai.com/api-keys" + ) + return + } } _uiState.value = _uiState.value.copy(isGeminiActive = true) + voiceService = VoiceServiceFactory.create(application) + + // Wire audio callbacks. A provider that manages its own audio (WebRTC) + // captures the mic and plays audio inside the peer connection, so the + // PCM pump stays idle for it. + if (!voiceService.managesOwnAudio) { + audioManager.onAudioCaptured = lambda@{ data -> + // Phone mode: mute mic while model speaks to prevent echo + if (streamingMode == StreamingMode.PHONE && voiceService.isModelSpeaking.value) return@lambda + voiceService.sendAudio(data) + } - // Wire audio callbacks - audioManager.onAudioCaptured = lambda@{ data -> - // Phone mode: mute mic while model speaks to prevent echo - if (streamingMode == StreamingMode.PHONE && geminiService.isModelSpeaking.value) return@lambda - geminiService.sendAudio(data) - } - - geminiService.onAudioReceived = { data -> - audioManager.playAudio(data) + voiceService.onAudioReceived = { data -> + audioManager.playAudio(data) + } } - geminiService.onInterrupted = { + voiceService.onInterrupted = { audioManager.stopPlayback() } - geminiService.onTurnComplete = { + voiceService.onTurnComplete = { _uiState.value = _uiState.value.copy(userTranscript = "") } - geminiService.onInputTranscription = { text -> + voiceService.onInputTranscription = { text -> _uiState.value = _uiState.value.copy( userTranscript = _uiState.value.userTranscript + text, aiTranscript = "" ) } - geminiService.onOutputTranscription = { text -> + voiceService.onOutputTranscription = { text -> _uiState.value = _uiState.value.copy( aiTranscript = _uiState.value.aiTranscript + text ) } - geminiService.onDisconnected = { reason -> + voiceService.onDisconnected = { reason -> if (_uiState.value.isGeminiActive) { stopSession() _uiState.value = _uiState.value.copy( @@ -109,15 +131,15 @@ class GeminiSessionViewModel : ViewModel() { // Wire tool call handling toolCallRouter = ToolCallRouter(openClawBridge, viewModelScope) - geminiService.onToolCall = { toolCall -> + voiceService.onToolCall = { toolCall -> for (call in toolCall.functionCalls) { - toolCallRouter?.handleToolCall(call) { response -> - geminiService.sendToolResponse(response) + toolCallRouter?.handleToolCall(call) { callId, name, result -> + voiceService.sendToolResponse(callId, name, result) } } } - geminiService.onToolCallCancellation = { cancellation -> + voiceService.onToolCallCancellation = { cancellation -> toolCallRouter?.cancelToolCalls(cancellation.ids) } @@ -126,8 +148,8 @@ class GeminiSessionViewModel : ViewModel() { while (isActive) { delay(100) _uiState.value = _uiState.value.copy( - connectionState = geminiService.connectionState.value, - isModelSpeaking = geminiService.isModelSpeaking.value, + connectionState = voiceService.connectionState.value, + isModelSpeaking = voiceService.isModelSpeaking.value, toolCallStatus = openClawBridge.lastToolCallStatus.value, openClawConnectionState = openClawBridge.connectionState.value, ) @@ -135,34 +157,36 @@ class GeminiSessionViewModel : ViewModel() { } // Connect to Gemini - geminiService.connect { setupOk -> + voiceService.connect { setupOk -> if (!setupOk) { - val msg = when (val state = geminiService.connectionState.value) { - is GeminiConnectionState.Error -> state.message + val msg = when (val state = voiceService.connectionState.value) { + is VoiceConnectionState.Error -> state.message else -> "Failed to connect to Gemini" } _uiState.value = _uiState.value.copy(errorMessage = msg) - geminiService.disconnect() + voiceService.disconnect() stateObservationJob?.cancel() _uiState.value = _uiState.value.copy( isGeminiActive = false, - connectionState = GeminiConnectionState.Disconnected + connectionState = VoiceConnectionState.Disconnected ) return@connect } - // Start mic capture + // Start mic capture (WebRTC providers capture inside the peer connection) try { - audioManager.startCapture() + if (!voiceService.managesOwnAudio) { + audioManager.startCapture() + } } catch (e: Exception) { _uiState.value = _uiState.value.copy( errorMessage = "Mic capture failed: ${e.message}" ) - geminiService.disconnect() + voiceService.disconnect() stateObservationJob?.cancel() _uiState.value = _uiState.value.copy( isGeminiActive = false, - connectionState = GeminiConnectionState.Disconnected + connectionState = VoiceConnectionState.Disconnected ) } @@ -170,8 +194,8 @@ class GeminiSessionViewModel : ViewModel() { if (SettingsManager.proactiveNotificationsEnabled) { eventClient.onNotification = { text -> val state = _uiState.value - if (state.isGeminiActive && state.connectionState == GeminiConnectionState.Ready) { - geminiService.sendTextMessage(text) + if (state.isGeminiActive && state.connectionState == VoiceConnectionState.Ready) { + voiceService.sendTextMessage(text) } } eventClient.connect() @@ -185,7 +209,7 @@ class GeminiSessionViewModel : ViewModel() { toolCallRouter?.cancelAll() toolCallRouter = null audioManager.stopCapture() - geminiService.disconnect() + voiceService.disconnect() stateObservationJob?.cancel() stateObservationJob = null _uiState.value = GeminiUiState() @@ -194,11 +218,11 @@ class GeminiSessionViewModel : ViewModel() { fun sendVideoFrameIfThrottled(bitmap: Bitmap) { if (!SettingsManager.videoStreamingEnabled) return if (!_uiState.value.isGeminiActive) return - if (_uiState.value.connectionState != GeminiConnectionState.Ready) return + if (_uiState.value.connectionState != VoiceConnectionState.Ready) return val now = System.currentTimeMillis() if (now - lastVideoFrameTime < GeminiConfig.VIDEO_FRAME_INTERVAL_MS) return lastVideoFrameTime = now - geminiService.sendVideoFrame(bitmap) + voiceService.sendVideoFrame(bitmap) } fun clearError() { diff --git a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/openclaw/ToolCallModels.kt b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/openclaw/ToolCallModels.kt index 696a0c8a6..9ad8a8a89 100644 --- a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/openclaw/ToolCallModels.kt +++ b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/openclaw/ToolCallModels.kt @@ -102,25 +102,41 @@ sealed class OpenClawConnectionState { // Tool Declarations (for Gemini setup message) object ToolDeclarations { + private const val EXECUTE_DESCRIPTION = "Your only way to take action. You have no memory, storage, or ability to do anything on your own -- use this tool for everything: sending messages, searching the web, adding to lists, setting reminders, creating notes, research, drafts, scheduling, smart home control, app interactions, or any request that goes beyond answering a question. When in doubt, use this tool." + fun allDeclarationsJSON(): JSONArray { return JSONArray().put(executeJSON()) } + // Same tool in the OpenAI Realtime function format (flat, with a type field). + fun openAIDeclarationsJSON(): JSONArray { + return JSONArray().put(JSONObject().apply { + put("type", "function") + put("name", "execute") + put("description", EXECUTE_DESCRIPTION) + put("parameters", executeParametersJSON()) + }) + } + private fun executeJSON(): JSONObject { return JSONObject().apply { put("name", "execute") - put("description", "Your only way to take action. You have no memory, storage, or ability to do anything on your own -- use this tool for everything: sending messages, searching the web, adding to lists, setting reminders, creating notes, research, drafts, scheduling, smart home control, app interactions, or any request that goes beyond answering a question. When in doubt, use this tool.") - put("parameters", JSONObject().apply { - put("type", "object") - put("properties", JSONObject().apply { - put("task", JSONObject().apply { - put("type", "string") - put("description", "Clear, detailed description of what to do. Include all relevant context: names, content, platforms, quantities, etc.") - }) + put("description", EXECUTE_DESCRIPTION) + put("parameters", executeParametersJSON()) + put("behavior", "BLOCKING") + } + } + + private fun executeParametersJSON(): JSONObject { + return JSONObject().apply { + put("type", "object") + put("properties", JSONObject().apply { + put("task", JSONObject().apply { + put("type", "string") + put("description", "Clear, detailed description of what to do. Include all relevant context: names, content, platforms, quantities, etc.") }) - put("required", JSONArray().put("task")) }) - put("behavior", "BLOCKING") + put("required", JSONArray().put("task")) } } } diff --git a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/openclaw/ToolCallRouter.kt b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/openclaw/ToolCallRouter.kt index 35337e145..5f240d6ca 100644 --- a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/openclaw/ToolCallRouter.kt +++ b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/openclaw/ToolCallRouter.kt @@ -4,8 +4,6 @@ import android.util.Log import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Job import kotlinx.coroutines.launch -import org.json.JSONArray -import org.json.JSONObject class ToolCallRouter( private val bridge: OpenClawBridge, @@ -21,7 +19,7 @@ class ToolCallRouter( fun handleToolCall( call: GeminiFunctionCall, - sendResponse: (JSONObject) -> Unit + sendResponse: (callId: String, name: String, result: ToolResult) -> Unit ) { val callId = call.id val callName = call.name @@ -35,7 +33,7 @@ class ToolCallRouter( "Tool execution is temporarily unavailable after $consecutiveFailures consecutive failures. " + "Please tell the user you cannot complete this action right now and suggest they check their OpenClaw gateway connection." ) - sendResponse(buildToolResponse(callId, callName, errorResult)) + sendResponse(callId, callName, errorResult) return } @@ -51,8 +49,7 @@ class ToolCallRouter( is ToolResult.Failure -> consecutiveFailures++ } - val response = buildToolResponse(callId, callName, result) - sendResponse(response) + sendResponse(callId, callName, result) } else { Log.d(TAG, "Task $callId was cancelled, skipping response") } @@ -82,20 +79,4 @@ class ToolCallRouter( inFlightJobs.clear() consecutiveFailures = 0 } - - private fun buildToolResponse( - callId: String, - name: String, - result: ToolResult - ): JSONObject { - return JSONObject().apply { - put("toolResponse", JSONObject().apply { - put("functionResponses", JSONArray().put(JSONObject().apply { - put("id", callId) - put("name", name) - put("response", result.toJSON()) - })) - }) - } - } } diff --git a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/settings/SettingsManager.kt b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/settings/SettingsManager.kt index dd8d2d26d..3decfc7d6 100644 --- a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/settings/SettingsManager.kt +++ b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/settings/SettingsManager.kt @@ -17,6 +17,15 @@ object SettingsManager { get() = prefs.getString("geminiAPIKey", null) ?: Secrets.geminiAPIKey set(value) = prefs.edit().putString("geminiAPIKey", value).apply() + // "gemini" (Live API over WebSocket) or "openai" (Realtime API over WebRTC) + var voiceProvider: String + get() = prefs.getString("voiceProvider", null) ?: "gemini" + set(value) = prefs.edit().putString("voiceProvider", value).apply() + + var openaiAPIKey: String + get() = prefs.getString("openaiAPIKey", null) ?: "" + set(value) = prefs.edit().putString("openaiAPIKey", value).apply() + var geminiSystemPrompt: String get() = prefs.getString("geminiSystemPrompt", null) ?: DEFAULT_SYSTEM_PROMPT set(value) = prefs.edit().putString("geminiSystemPrompt", value).apply() diff --git a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/ui/GeminiOverlayView.kt b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/ui/GeminiOverlayView.kt index 8cfa09cfe..ae4f0280f 100644 --- a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/ui/GeminiOverlayView.kt +++ b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/ui/GeminiOverlayView.kt @@ -31,7 +31,7 @@ import androidx.compose.ui.text.font.FontFamily import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp -import com.meta.wearable.dat.externalsampleapps.cameraaccess.gemini.GeminiConnectionState +import com.meta.wearable.dat.externalsampleapps.cameraaccess.voice.VoiceConnectionState import com.meta.wearable.dat.externalsampleapps.cameraaccess.gemini.GeminiUiState import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.OpenClawConnectionState import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.ToolCallStatus @@ -77,7 +77,7 @@ fun GeminiOverlay( @Composable fun GeminiStatusBar( - connectionState: GeminiConnectionState, + connectionState: VoiceConnectionState, openClawState: OpenClawConnectionState, modifier: Modifier = Modifier, ) { @@ -88,11 +88,11 @@ fun GeminiStatusBar( StatusPill( label = "AI", color = when (connectionState) { - is GeminiConnectionState.Ready -> Color(0xFF4CAF50) - is GeminiConnectionState.Connecting, - is GeminiConnectionState.SettingUp -> Color(0xFFFF9800) - is GeminiConnectionState.Error -> Color(0xFFF44336) - is GeminiConnectionState.Disconnected -> Color(0xFF9E9E9E) + is VoiceConnectionState.Ready -> Color(0xFF4CAF50) + is VoiceConnectionState.Connecting, + is VoiceConnectionState.SettingUp -> Color(0xFFFF9800) + is VoiceConnectionState.Error -> Color(0xFFF44336) + is VoiceConnectionState.Disconnected -> Color(0xFF9E9E9E) }, ) diff --git a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/ui/SettingsScreen.kt b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/ui/SettingsScreen.kt index dd9133636..3ad422f25 100644 --- a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/ui/SettingsScreen.kt +++ b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/ui/SettingsScreen.kt @@ -43,6 +43,8 @@ fun SettingsScreen( modifier: Modifier = Modifier, ) { var geminiAPIKey by remember { mutableStateOf(SettingsManager.geminiAPIKey) } + var useOpenAIRealtime by remember { mutableStateOf(SettingsManager.voiceProvider == "openai") } + var openaiAPIKey by remember { mutableStateOf(SettingsManager.openaiAPIKey) } var systemPrompt by remember { mutableStateOf(SettingsManager.geminiSystemPrompt) } var openClawHost by remember { mutableStateOf(SettingsManager.openClawHost) } var openClawPort by remember { mutableStateOf(SettingsManager.openClawPort.toString()) } @@ -55,6 +57,8 @@ fun SettingsScreen( fun save() { SettingsManager.geminiAPIKey = geminiAPIKey.trim() + SettingsManager.voiceProvider = if (useOpenAIRealtime) "openai" else "gemini" + SettingsManager.openaiAPIKey = openaiAPIKey.trim() SettingsManager.geminiSystemPrompt = systemPrompt.trim() SettingsManager.openClawHost = openClawHost.trim() openClawPort.trim().toIntOrNull()?.let { SettingsManager.openClawPort = it } @@ -67,6 +71,8 @@ fun SettingsScreen( fun reload() { geminiAPIKey = SettingsManager.geminiAPIKey + useOpenAIRealtime = SettingsManager.voiceProvider == "openai" + openaiAPIKey = SettingsManager.openaiAPIKey systemPrompt = SettingsManager.geminiSystemPrompt openClawHost = SettingsManager.openClawHost openClawPort = SettingsManager.openClawPort.toString() @@ -107,6 +113,33 @@ fun SettingsScreen( placeholder = "Enter Gemini API key", ) + // Voice provider section + SectionHeader("Voice Provider") + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.SpaceBetween, + verticalAlignment = androidx.compose.ui.Alignment.CenterVertically, + ) { + Column { + Text("OpenAI Realtime (WebRTC)", style = MaterialTheme.typography.bodyLarge) + Text( + "Use the OpenAI Realtime API over WebRTC instead of Gemini Live. Takes effect on next session.", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + Switch( + checked = useOpenAIRealtime, + onCheckedChange = { useOpenAIRealtime = it }, + ) + } + MonoTextField( + value = openaiAPIKey, + onValueChange = { openaiAPIKey = it }, + label = "OpenAI API Key", + placeholder = "Enter OpenAI API key", + ) + SectionHeader("System Prompt") OutlinedTextField( value = systemPrompt, diff --git a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/voice/RealtimeVoiceService.kt b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/voice/RealtimeVoiceService.kt new file mode 100644 index 000000000..c87fa8580 --- /dev/null +++ b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/voice/RealtimeVoiceService.kt @@ -0,0 +1,60 @@ +package com.meta.wearable.dat.externalsampleapps.cameraaccess.voice + +import android.content.Context +import android.graphics.Bitmap +import com.meta.wearable.dat.externalsampleapps.cameraaccess.gemini.GeminiLiveService +import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.GeminiToolCall +import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.GeminiToolCallCancellation +import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.ToolResult +import com.meta.wearable.dat.externalsampleapps.cameraaccess.settings.SettingsManager +import com.meta.wearable.dat.externalsampleapps.cameraaccess.voice.openai.OpenAIRealtimeService +import kotlinx.coroutines.flow.StateFlow + +// Extracted from GeminiLiveService's public surface so alternate realtime voice +// backends (e.g. WebRTC-based providers) can plug into GeminiSessionViewModel. +// Tool-call types keep their Gemini names for now; they act as the app's neutral +// internal representation and other providers map into them. +interface RealtimeVoiceService { + val connectionState: StateFlow + val isModelSpeaking: StateFlow + + // true when the provider owns mic capture and speaker playback end to end + // (WebRTC). false when the caller must pump PCM through sendAudio() and + // play back audio delivered via onAudioReceived. + val managesOwnAudio: Boolean + + var onAudioReceived: ((ByteArray) -> Unit)? + var onTurnComplete: (() -> Unit)? + var onInterrupted: (() -> Unit)? + var onDisconnected: ((String?) -> Unit)? + var onInputTranscription: ((String) -> Unit)? + var onOutputTranscription: ((String) -> Unit)? + var onToolCall: ((GeminiToolCall) -> Unit)? + var onToolCallCancellation: ((GeminiToolCallCancellation) -> Unit)? + + fun connect(callback: (Boolean) -> Unit) + fun disconnect() + fun sendAudio(data: ByteArray) + fun sendVideoFrame(bitmap: Bitmap) + fun sendToolResponse(callId: String, name: String, result: ToolResult) + fun sendTextMessage(text: String) +} + +enum class VoiceProvider(val id: String) { + GEMINI("gemini"), + OPENAI_REALTIME("openai"); + + companion object { + fun fromId(id: String): VoiceProvider = + entries.firstOrNull { it.id == id } ?: GEMINI + } +} + +object VoiceServiceFactory { + fun current(): VoiceProvider = VoiceProvider.fromId(SettingsManager.voiceProvider) + + fun create(context: Context): RealtimeVoiceService = when (current()) { + VoiceProvider.GEMINI -> GeminiLiveService() + VoiceProvider.OPENAI_REALTIME -> OpenAIRealtimeService(context.applicationContext) + } +} diff --git a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/voice/VoiceConnectionState.kt b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/voice/VoiceConnectionState.kt new file mode 100644 index 000000000..68fad4cac --- /dev/null +++ b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/voice/VoiceConnectionState.kt @@ -0,0 +1,11 @@ +package com.meta.wearable.dat.externalsampleapps.cameraaccess.voice + +// Provider-neutral connection state for realtime voice sessions. +// (Moved from gemini/GeminiLiveService.kt.) +sealed class VoiceConnectionState { + data object Disconnected : VoiceConnectionState() + data object Connecting : VoiceConnectionState() + data object SettingUp : VoiceConnectionState() + data object Ready : VoiceConnectionState() + data class Error(val message: String) : VoiceConnectionState() +} diff --git a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/voice/openai/OpenAIRealtimeConfig.kt b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/voice/openai/OpenAIRealtimeConfig.kt new file mode 100644 index 000000000..8458efa4d --- /dev/null +++ b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/voice/openai/OpenAIRealtimeConfig.kt @@ -0,0 +1,27 @@ +package com.meta.wearable.dat.externalsampleapps.cameraaccess.voice.openai + +import com.meta.wearable.dat.externalsampleapps.cameraaccess.settings.SettingsManager + +object OpenAIRealtimeConfig { + // Ephemeral-token flow: mint a client secret server-side style with the API + // key, then POST the SDP offer with the ephemeral key. + // https://developers.openai.com/api/docs/guides/realtime-webrtc + const val CLIENT_SECRETS_URL = "https://api.openai.com/v1/realtime/client_secrets" + const val CALLS_URL = "https://api.openai.com/v1/realtime/calls" + + const val MODEL = "gpt-realtime-2.1" + const val VOICE = "marin" + const val TRANSCRIPTION_MODEL = "gpt-4o-mini-transcribe" + + const val VIDEO_JPEG_QUALITY = 50 + + val apiKey: String + get() = SettingsManager.openaiAPIKey + + // The system prompt is provider-neutral; reuse the existing setting. + val systemInstruction: String + get() = SettingsManager.geminiSystemPrompt + + val isConfigured: Boolean + get() = apiKey.isNotEmpty() && !apiKey.startsWith("YOUR_") +} diff --git a/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/voice/openai/OpenAIRealtimeService.kt b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/voice/openai/OpenAIRealtimeService.kt new file mode 100644 index 000000000..5d84b2565 --- /dev/null +++ b/samples/CameraAccessAndroid/app/src/main/java/com/meta/wearable/dat/externalsampleapps/cameraaccess/voice/openai/OpenAIRealtimeService.kt @@ -0,0 +1,502 @@ +package com.meta.wearable.dat.externalsampleapps.cameraaccess.voice.openai + +import android.content.Context +import android.graphics.Bitmap +import android.util.Base64 +import android.util.Log +import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.GeminiFunctionCall +import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.GeminiToolCall +import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.GeminiToolCallCancellation +import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.ToolDeclarations +import com.meta.wearable.dat.externalsampleapps.cameraaccess.openclaw.ToolResult +import com.meta.wearable.dat.externalsampleapps.cameraaccess.voice.RealtimeVoiceService +import com.meta.wearable.dat.externalsampleapps.cameraaccess.voice.VoiceConnectionState +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import java.nio.charset.StandardCharsets +import java.util.Timer +import java.util.TimerTask +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicBoolean +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import org.json.JSONArray +import org.json.JSONObject +import org.webrtc.DataChannel +import org.webrtc.IceCandidate +import org.webrtc.MediaConstraints +import org.webrtc.MediaStream +import org.webrtc.PeerConnection +import org.webrtc.PeerConnectionFactory +import org.webrtc.RtpReceiver +import org.webrtc.SdpObserver +import org.webrtc.SessionDescription +import org.webrtc.audio.JavaAudioDeviceModule + +// Realtime voice over WebRTC against the OpenAI Realtime API. +// +// Unlike the Gemini Live path, WebRTC owns the microphone and speaker: capture, +// playback, echo cancellation, jitter buffering, and Opus encoding all happen +// inside the peer connection (managesOwnAudio = true), so AudioManager's PCM +// pump stays off. Model/tool events flow over the "oai-events" data channel. +// +// Vision: continuous 1fps frame push (the Gemini approach) would flood the +// conversation context, so the latest camera frame is cached and attached as an +// image item once per user turn, when the server reports speech start. +class OpenAIRealtimeService(private val context: Context) : RealtimeVoiceService { + companion object { + private const val TAG = "OpenAIRealtimeService" + private const val ICE_GATHERING_TIMEOUT_MS = 2000L + private const val CONNECT_TIMEOUT_MS = 20000L + private const val DATA_CHANNEL_LABEL = "oai-events" + } + + private val _connectionState = + MutableStateFlow(VoiceConnectionState.Disconnected) + override val connectionState: StateFlow = _connectionState.asStateFlow() + + private val _isModelSpeaking = MutableStateFlow(false) + override val isModelSpeaking: StateFlow = _isModelSpeaking.asStateFlow() + + override val managesOwnAudio: Boolean = true + + override var onAudioReceived: ((ByteArray) -> Unit)? = null // unused: WebRTC plays audio itself + override var onTurnComplete: (() -> Unit)? = null + override var onInterrupted: (() -> Unit)? = null + override var onDisconnected: ((String?) -> Unit)? = null + override var onInputTranscription: ((String) -> Unit)? = null + override var onOutputTranscription: ((String) -> Unit)? = null + override var onToolCall: ((GeminiToolCall) -> Unit)? = null + override var onToolCallCancellation: ((GeminiToolCallCancellation) -> Unit)? = null + + private var peerConnectionFactory: PeerConnectionFactory? = null + private var peerConnection: PeerConnection? = null + private var audioDeviceModule: JavaAudioDeviceModule? = null + private var dataChannel: DataChannel? = null + + private val httpClient = OkHttpClient() + private val sendExecutor = Executors.newSingleThreadExecutor() + private var connectCallback: ((Boolean) -> Unit)? = null + private var timeoutTimer: Timer? = null + private val offerPosted = AtomicBoolean(false) + private var ephemeralKey: String? = null + + @Volatile + private var pendingFrameBase64: String? = null + + override fun connect(callback: (Boolean) -> Unit) { + if (!OpenAIRealtimeConfig.isConfigured) { + _connectionState.value = VoiceConnectionState.Error("No OpenAI API key configured") + callback(false) + return + } + + _connectionState.value = VoiceConnectionState.Connecting + connectCallback = callback + offerPosted.set(false) + + timeoutTimer = Timer().apply { + schedule(object : TimerTask() { + override fun run() { + if (_connectionState.value == VoiceConnectionState.Connecting + || _connectionState.value == VoiceConnectionState.SettingUp) { + Log.e(TAG, "Connection timed out") + failConnect("Connection timed out") + } + } + }, CONNECT_TIMEOUT_MS) + } + + Thread({ + try { + ephemeralKey = mintClientSecret() + setupPeerConnection() + } catch (e: Exception) { + Log.e(TAG, "Connect failed: ${e.message}") + failConnect(e.message ?: "Connect failed") + } + }, "openai-connect").start() + } + + override fun disconnect() { + timeoutTimer?.cancel() + timeoutTimer = null + onToolCall = null + onToolCallCancellation = null + teardown() + _connectionState.value = VoiceConnectionState.Disconnected + _isModelSpeaking.value = false + resolveConnect(false) + } + + // WebRTC captures the microphone itself; PCM pushed by the caller is ignored. + override fun sendAudio(data: ByteArray) {} + + override fun sendVideoFrame(bitmap: Bitmap) { + if (_connectionState.value != VoiceConnectionState.Ready) return + val baos = ByteArrayOutputStream() + bitmap.compress(Bitmap.CompressFormat.JPEG, OpenAIRealtimeConfig.VIDEO_JPEG_QUALITY, baos) + pendingFrameBase64 = Base64.encodeToString(baos.toByteArray(), Base64.NO_WRAP) + } + + override fun sendToolResponse(callId: String, name: String, result: ToolResult) { + sendEvent(JSONObject().apply { + put("type", "conversation.item.create") + put("item", JSONObject().apply { + put("type", "function_call_output") + put("call_id", callId) + put("output", result.toJSON().toString()) + }) + }) + sendEvent(JSONObject().put("type", "response.create")) + } + + override fun sendTextMessage(text: String) { + if (_connectionState.value != VoiceConnectionState.Ready) return + sendEvent(JSONObject().apply { + put("type", "conversation.item.create") + put("item", JSONObject().apply { + put("type", "message") + put("role", "user") + put("content", JSONArray().put(JSONObject().apply { + put("type", "input_text") + put("text", text) + })) + }) + }) + sendEvent(JSONObject().put("type", "response.create")) + } + + // Connection setup + + private fun mintClientSecret(): String { + val session = JSONObject().apply { + put("session", JSONObject().apply { + put("type", "realtime") + put("model", OpenAIRealtimeConfig.MODEL) + put("instructions", OpenAIRealtimeConfig.systemInstruction) + put("audio", JSONObject().apply { + put("input", JSONObject().apply { + put("transcription", JSONObject().apply { + put("model", OpenAIRealtimeConfig.TRANSCRIPTION_MODEL) + }) + }) + put("output", JSONObject().apply { + put("voice", OpenAIRealtimeConfig.VOICE) + }) + }) + put("tools", ToolDeclarations.openAIDeclarationsJSON()) + }) + } + + val request = Request.Builder() + .url(OpenAIRealtimeConfig.CLIENT_SECRETS_URL) + .header("Authorization", "Bearer ${OpenAIRealtimeConfig.apiKey}") + .post(session.toString().toRequestBody("application/json".toMediaType())) + .build() + + httpClient.newCall(request).execute().use { response -> + val body = response.body?.string() ?: "" + if (!response.isSuccessful) { + throw IllegalStateException("client_secrets HTTP ${response.code}: ${body.take(200)}") + } + val value = JSONObject(body).optString("value", "") + if (value.isEmpty()) throw IllegalStateException("client_secrets response missing value") + return value + } + } + + private fun setupPeerConnection() { + PeerConnectionFactory.initialize( + PeerConnectionFactory.InitializationOptions.builder(context) + .setEnableInternalTracer(false) + .createInitializationOptions() + ) + + val adm = JavaAudioDeviceModule.builder(context) + .setUseHardwareAcousticEchoCanceler(true) + .setUseHardwareNoiseSuppressor(true) + .createAudioDeviceModule() + audioDeviceModule = adm + + val factory = PeerConnectionFactory.builder() + .setAudioDeviceModule(adm) + .createPeerConnectionFactory() + peerConnectionFactory = factory + + val rtcConfig = PeerConnection.RTCConfiguration( + listOf(PeerConnection.IceServer.builder("stun:stun.l.google.com:19302").createIceServer()) + ) + rtcConfig.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN + + val pc = factory.createPeerConnection(rtcConfig, object : PeerConnection.Observer { + override fun onSignalingChange(state: PeerConnection.SignalingState?) {} + + override fun onIceConnectionChange(state: PeerConnection.IceConnectionState?) { + Log.d(TAG, "ICE connection state: $state") + if (state == PeerConnection.IceConnectionState.FAILED) { + handleDropped("ICE connection failed") + } + } + + override fun onIceConnectionReceivingChange(receiving: Boolean) {} + + override fun onIceGatheringChange(state: PeerConnection.IceGatheringState?) { + Log.d(TAG, "ICE gathering state: $state") + if (state == PeerConnection.IceGatheringState.COMPLETE) { + postOfferOnce() + } + } + + override fun onIceCandidate(candidate: IceCandidate?) {} + override fun onIceCandidatesRemoved(candidates: Array?) {} + override fun onAddStream(stream: MediaStream?) {} + override fun onRemoveStream(stream: MediaStream?) {} + override fun onDataChannel(channel: DataChannel?) {} + override fun onRenegotiationNeeded() {} + override fun onAddTrack(receiver: RtpReceiver?, streams: Array?) {} + }) ?: throw IllegalStateException("Failed to create peer connection") + peerConnection = pc + + val audioSource = factory.createAudioSource(MediaConstraints()) + val audioTrack = factory.createAudioTrack("mic0", audioSource).apply { setEnabled(true) } + pc.addTrack(audioTrack, listOf("mic")) + + dataChannel = pc.createDataChannel(DATA_CHANNEL_LABEL, DataChannel.Init()) + dataChannel?.registerObserver(object : DataChannel.Observer { + override fun onBufferedAmountChange(previousAmount: Long) {} + + override fun onStateChange() { + val state = dataChannel?.state() + Log.d(TAG, "Data channel state: $state") + when (state) { + DataChannel.State.OPEN -> { + _connectionState.value = VoiceConnectionState.Ready + resolveConnect(true) + } + DataChannel.State.CLOSED -> handleDropped("Data channel closed") + else -> {} + } + } + + override fun onMessage(buffer: DataChannel.Buffer?) { + buffer ?: return + val bytes = ByteArray(buffer.data.remaining()) + buffer.data.get(bytes) + handleEvent(String(bytes, StandardCharsets.UTF_8)) + } + }) + + val constraints = MediaConstraints().apply { + mandatory.add(MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")) + } + pc.createOffer(object : SdpObserver { + override fun onCreateSuccess(sdp: SessionDescription?) { + sdp ?: return failConnect("Empty SDP offer") + pc.setLocalDescription(object : SdpObserver { + override fun onSetSuccess() { + // Post after ICE gathering completes, or after a short + // timeout with whatever candidates are already in the SDP. + Timer().schedule(object : TimerTask() { + override fun run() { postOfferOnce() } + }, ICE_GATHERING_TIMEOUT_MS) + } + override fun onSetFailure(error: String?) { + failConnect("setLocalDescription failed: $error") + } + override fun onCreateSuccess(p0: SessionDescription?) {} + override fun onCreateFailure(p0: String?) {} + }, sdp) + } + override fun onCreateFailure(error: String?) { + failConnect("createOffer failed: $error") + } + override fun onSetSuccess() {} + override fun onSetFailure(p0: String?) {} + }, constraints) + } + + private fun postOfferOnce() { + if (offerPosted.getAndSet(true)) return + val offerSdp = peerConnection?.localDescription?.description + ?: return failConnect("No local SDP") + + Thread({ + try { + val request = Request.Builder() + .url(OpenAIRealtimeConfig.CALLS_URL) + .header("Authorization", "Bearer ${ephemeralKey ?: ""}") + .post(offerSdp.toRequestBody("application/sdp".toMediaType())) + .build() + + val answerSdp = httpClient.newCall(request).execute().use { response -> + val body = response.body?.string() ?: "" + if (!response.isSuccessful) { + throw IllegalStateException("calls HTTP ${response.code}: ${body.take(200)}") + } + body + } + + _connectionState.value = VoiceConnectionState.SettingUp + peerConnection?.setRemoteDescription(object : SdpObserver { + override fun onSetSuccess() { + Log.d(TAG, "Remote description set; waiting for data channel") + } + override fun onSetFailure(error: String?) { + failConnect("setRemoteDescription failed: $error") + } + override fun onCreateSuccess(p0: SessionDescription?) {} + override fun onCreateFailure(p0: String?) {} + }, SessionDescription(SessionDescription.Type.ANSWER, answerSdp)) + } catch (e: Exception) { + Log.e(TAG, "SDP exchange failed: ${e.message}") + failConnect(e.message ?: "SDP exchange failed") + } + }, "openai-sdp").start() + } + + // Event handling + + private fun handleEvent(text: String) { + try { + val json = JSONObject(text) + when (val type = json.optString("type", "")) { + "session.created", "session.updated" -> { + Log.d(TAG, type) + } + + "input_audio_buffer.speech_started" -> { + // Barge-in: the server clears the output buffer; mirror state. + _isModelSpeaking.value = false + onInterrupted?.invoke() + sendPendingFrame() + } + + "conversation.item.input_audio_transcription.completed" -> { + val transcript = json.optString("transcript", "") + if (transcript.isNotEmpty()) { + Log.d(TAG, "You: $transcript") + onInputTranscription?.invoke(transcript) + } + } + + // Event renamed between Realtime API versions; accept both. + "response.output_audio_transcript.delta", "response.audio_transcript.delta" -> { + val delta = json.optString("delta", "") + if (delta.isNotEmpty()) onOutputTranscription?.invoke(delta) + } + + "output_audio_buffer.started" -> _isModelSpeaking.value = true + + "output_audio_buffer.stopped", "output_audio_buffer.cleared" -> { + _isModelSpeaking.value = false + } + + "response.function_call_arguments.done" -> { + val callId = json.optString("call_id", "") + val name = json.optString("name", "") + if (callId.isNotEmpty() && name.isNotEmpty()) { + val args = mutableMapOf() + try { + val argsObj = JSONObject(json.optString("arguments", "{}")) + for (key in argsObj.keys()) args[key] = argsObj.opt(key) + } catch (e: Exception) { + Log.e(TAG, "Bad tool args: ${e.message}") + } + Log.d(TAG, "Tool call: $name (id: $callId)") + onToolCall?.invoke( + GeminiToolCall(listOf(GeminiFunctionCall(callId, name, args))) + ) + } + } + + "response.done" -> { + onTurnComplete?.invoke() + } + + "error" -> { + val message = json.optJSONObject("error")?.optString("message") ?: text.take(200) + Log.e(TAG, "Server error: $message") + if (_connectionState.value != VoiceConnectionState.Ready) { + failConnect(message) + } + } + } + } catch (e: Exception) { + Log.e(TAG, "Error parsing event: ${e.message}") + } + } + + private fun sendPendingFrame() { + val base64 = pendingFrameBase64 ?: return + pendingFrameBase64 = null + sendEvent(JSONObject().apply { + put("type", "conversation.item.create") + put("item", JSONObject().apply { + put("type", "message") + put("role", "user") + put("content", JSONArray().put(JSONObject().apply { + put("type", "input_image") + put("image_url", "data:image/jpeg;base64,$base64") + })) + }) + }) + } + + private fun sendEvent(json: JSONObject) { + sendExecutor.execute { + val channel = dataChannel ?: return@execute + if (channel.state() != DataChannel.State.OPEN) return@execute + val bytes = json.toString().toByteArray(StandardCharsets.UTF_8) + channel.send(DataChannel.Buffer(ByteBuffer.wrap(bytes), false)) + } + } + + // Lifecycle helpers + + private fun resolveConnect(success: Boolean) { + val cb = connectCallback + connectCallback = null // null out BEFORE invoking to prevent re-entrancy + timeoutTimer?.cancel() + timeoutTimer = null + cb?.invoke(success) + } + + private fun failConnect(message: String) { + _connectionState.value = VoiceConnectionState.Error(message) + _isModelSpeaking.value = false + // failConnect can fire from WebRTC observer callbacks, and + // PeerConnection.close() blocks until callbacks return -- tear down on + // a separate thread to avoid deadlocking libwebrtc's signaling thread. + Thread({ teardown() }, "openai-teardown").start() + resolveConnect(false) + } + + private fun handleDropped(reason: String) { + if (_connectionState.value == VoiceConnectionState.Disconnected) return + val wasReady = _connectionState.value == VoiceConnectionState.Ready + _connectionState.value = VoiceConnectionState.Disconnected + _isModelSpeaking.value = false + resolveConnect(false) + if (wasReady) onDisconnected?.invoke(reason) + } + + private fun teardown() { + pendingFrameBase64 = null + ephemeralKey = null + dataChannel?.unregisterObserver() + dataChannel?.close() + dataChannel = null + peerConnection?.close() + peerConnection = null + peerConnectionFactory?.dispose() + peerConnectionFactory = null + audioDeviceModule?.release() + audioDeviceModule = null + } +}