From 27f9219310fa807b2ef489c2b22ed4eb643f7929 Mon Sep 17 00:00:00 2001 From: Usta Date: Mon, 3 Aug 2026 03:47:10 +0900 Subject: [PATCH] Facade: let the app supply program audio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vendored audio path SELECTS a source, it never mixes: AudioUnit's captureOutput takes one buffer straight to the encoder, and selecting a buffered source makes the microphone path bail out entirely. So "the streamer's voice AND the alert/TTS audio" cannot be produced inside the engine — there is nowhere for a second stream to join. Rather than fork the vendor to add a mixing stage, hand the job to the app, which is where the other audio already lives (speech synthesis, alert sounds, the captions tap): it owns one graph, mixes what belongs on the broadcast, and pushes the result through the new ProgramAudioSink seam. The engine treats it as the program source and encodes it. Kept off the StreamEngine protocol on purpose — it trades in CMSampleBuffer, and that protocol stays free of media types so a fake needs zero hardware. Same reasoning as CameraPreviewSource. The takeover is re-established after every setNetStream rebuild, alongside the mute re-apply: without that, go-live silently reverts the broadcast to bare mic. Releasing it re-attaches the default audio device rather than passing a nil buffered id, which would attach nothing and leave the stream silent. Tests: fake conformance records the takeover and buffer counts; 37/9 green on the simulator, containment clean. Co-Authored-By: Claude Fable 5 --- .../IRLStreamKit/Facade/CameraPreview.swift | 2 + .../IRLStreamKit/Facade/IRLStreamEngine.swift | 39 +++++++++ .../IRLStreamKit/Facade/ProgramAudio.swift | 48 ++++++++++ .../FakeStreamEngine.swift | 26 ++++++ .../IRLStreamKitTests/ProgramAudioTests.swift | 87 +++++++++++++++++++ 5 files changed, 202 insertions(+) create mode 100644 Sources/IRLStreamKit/Facade/ProgramAudio.swift create mode 100644 Tests/IRLStreamKitTests/ProgramAudioTests.swift diff --git a/Sources/IRLStreamKit/Facade/CameraPreview.swift b/Sources/IRLStreamKit/Facade/CameraPreview.swift index 99e4832..704a343 100644 --- a/Sources/IRLStreamKit/Facade/CameraPreview.swift +++ b/Sources/IRLStreamKit/Facade/CameraPreview.swift @@ -11,6 +11,8 @@ public protocol CameraPreviewSource: AnyObject { func makePreviewUIView() -> UIView } +extension IRLStreamEngine: ProgramAudioSink {} + extension IRLStreamEngine: CameraPreviewSource { public func makePreviewUIView() -> UIView { internalPreviewView diff --git a/Sources/IRLStreamKit/Facade/IRLStreamEngine.swift b/Sources/IRLStreamKit/Facade/IRLStreamEngine.swift index c3d6e23..e53095e 100644 --- a/Sources/IRLStreamKit/Facade/IRLStreamEngine.swift +++ b/Sources/IRLStreamKit/Facade/IRLStreamEngine.swift @@ -4,6 +4,7 @@ // service. import AVFoundation +import CoreMedia import Foundation import IRLTPBonding import Observation @@ -36,6 +37,10 @@ public final class IRLStreamEngine: StreamEngine { @ObservationIgnored private var desiredZoomX: [CameraSelection: Float] = [:] // Moblin's default zoom ramp speed (database.zoom.speed midpoint). private static let zoomRampRate: Float = 5.0 + // App-supplied program audio, if the consumer took it over. Like mute and + // the camera attach, this is lost whenever setNetStream rebuilds the + // Processor, so it is re-established after every rebuild. + @ObservationIgnored private var programAudio: (id: UUID, targetLatency: TimeInterval)? // streamTotal() resets on every (re)connect; accumulate across reconnects. @ObservationIgnored private var totalBytesBase: Int64 = 0 @ObservationIgnored let internalPreviewView: PreviewView @@ -309,6 +314,32 @@ public final class IRLStreamEngine: StreamEngine { device.position == .front ? .front : .back } + // MARK: - Program audio + + public var isProgramAudioActive: Bool { programAudio != nil } + + public func startProgramAudio(targetLatency: TimeInterval) { + guard programAudio == nil else { return } + let id = UUID() + programAudio = (id, targetLatency) + media.addBufferedAudio(cameraId: id, name: "Program", latency: targetLatency) + media.attachBufferedAudio(cameraId: id) + } + + public func appendProgramAudio(_ sampleBuffer: CMSampleBuffer) { + guard let programAudio else { return } + media.appendBufferedAudioSampleBuffer(cameraId: programAudio.id, sampleBuffer: sampleBuffer) + } + + public func stopProgramAudio() { + guard let programAudio else { return } + self.programAudio = nil + // Back to the microphone. `attachBufferedAudio(cameraId: nil)` would + // attach NOTHING (device nil, buffer nil) and leave the stream silent. + media.attachDefaultAudioDevice(builtinDelay: 0) + media.removeBufferedAudio(cameraId: programAudio.id) + } + // MARK: - Signal handling (single consumer preserves callback ordering) private func handle(_ stamped: StampedSignal) { @@ -411,6 +442,14 @@ public final class IRLStreamEngine: StreamEngine { // The fresh Processor lost the mute flag (AudioUnit.muted defaults to // false) — re-apply the user's intent, mirroring Moblin's updateMute(). media.setMute(on: desiredMicMuted) + // setNetStream re-attached the built-in mic; if the app owns program + // audio, take it back or the broadcast silently reverts to bare mic + // at go-live — exactly the class of bug the mute re-apply exists for. + if let programAudio { + media.addBufferedAudio(cameraId: programAudio.id, name: "Program", + latency: programAudio.targetLatency) + media.attachBufferedAudio(cameraId: programAudio.id) + } // Rebind the preview drawable and start the new Processor (mirrors // ModelStream.attachStream). if let processor = media.getProcessor() { diff --git a/Sources/IRLStreamKit/Facade/ProgramAudio.swift b/Sources/IRLStreamKit/Facade/ProgramAudio.swift new file mode 100644 index 0000000..c6af292 --- /dev/null +++ b/Sources/IRLStreamKit/Facade/ProgramAudio.swift @@ -0,0 +1,48 @@ +import CoreMedia +import Foundation + +/// Program audio supplied by the CONSUMER instead of the built-in microphone. +/// +/// Why this exists: the vendored audio path *selects* a source, it never +/// mixes. `AudioUnit.captureOutput` takes one buffer and hands it straight to +/// the encoder, and selecting a buffered source makes the microphone path +/// bail out entirely. So "the streamer's voice **and** the alert/TTS audio" +/// cannot be produced inside the engine at all — there is nowhere for a +/// second stream to join. +/// +/// Rather than fork the vendor to add a mixing stage, this hands the job to +/// the app, which is where the other audio already lives (speech synthesis, +/// alert sounds, the captions tap). The app owns one audio graph, mixes what +/// belongs on the broadcast, and pushes the result here; the engine treats it +/// as the program source and encodes it. +/// +/// Deliberately NOT part of `StreamEngine`: it trades in `CMSampleBuffer`, +/// and the main protocol is kept free of media types so a fake needs zero +/// hardware. Same reasoning as `CameraPreviewSource`. +@MainActor +public protocol ProgramAudioSink: AnyObject { + /// Whether app-supplied audio is currently the program source. + var isProgramAudioActive: Bool { get } + + /// Take over program audio. The built-in microphone stops being the + /// program source — everything the viewer hears now comes from + /// `appendProgramAudio`, so the caller must include the mic in its mix. + /// + /// `targetLatency` is the jitter buffer the engine keeps for this source; + /// it trades delay for tolerance of an irregular feed. + func startProgramAudio(targetLatency: TimeInterval) + + /// Feed one buffer of mixed program audio. + func appendProgramAudio(_ sampleBuffer: CMSampleBuffer) + + /// Hand program audio back to the built-in microphone. + func stopProgramAudio() +} + +public extension ProgramAudioSink { + /// 200 ms: enough to absorb a synthesis burst without a perceptible lag + /// between what the streamer says and what the viewer hears. + func startProgramAudio() { + startProgramAudio(targetLatency: 0.2) + } +} diff --git a/Sources/IRLStreamKitTestSupport/FakeStreamEngine.swift b/Sources/IRLStreamKitTestSupport/FakeStreamEngine.swift index 12d8f8b..821f244 100644 --- a/Sources/IRLStreamKitTestSupport/FakeStreamEngine.swift +++ b/Sources/IRLStreamKitTestSupport/FakeStreamEngine.swift @@ -3,6 +3,7 @@ // engine and enforces the same phase machine, so fake and real cannot // diverge on state derivation or lifecycle contract. +import CoreMedia import Foundation import IRLStreamKit import Observation @@ -34,6 +35,11 @@ public final class FakeStreamEngine: StreamEngine { @ObservationIgnored private let broadcaster = EventBroadcaster() + /// Set while the consumer owns program audio; nil means the built-in mic. + public internal(set) var programAudioLatency: TimeInterval? + /// How many mixed buffers the consumer pushed while it owned program audio. + public internal(set) var programAudioBufferCount = 0 + public init() {} /// Tests drive the world: applies the same package reducer synchronously @@ -162,3 +168,23 @@ public final class FakeStreamEngine: StreamEngine { emit(.stabilizationChanged(mode)) } } + +/// Program-audio conformance for the fake: records the takeover and every +/// buffer count, so a consumer's mixer can be tested with zero hardware. +extension FakeStreamEngine: ProgramAudioSink { + public var isProgramAudioActive: Bool { programAudioLatency != nil } + + public func startProgramAudio(targetLatency: TimeInterval) { + guard programAudioLatency == nil else { return } + programAudioLatency = targetLatency + } + + public func appendProgramAudio(_: CMSampleBuffer) { + guard programAudioLatency != nil else { return } + programAudioBufferCount += 1 + } + + public func stopProgramAudio() { + programAudioLatency = nil + } +} diff --git a/Tests/IRLStreamKitTests/ProgramAudioTests.swift b/Tests/IRLStreamKitTests/ProgramAudioTests.swift new file mode 100644 index 0000000..6d9e9c7 --- /dev/null +++ b/Tests/IRLStreamKitTests/ProgramAudioTests.swift @@ -0,0 +1,87 @@ +import CoreMedia +import Foundation +import IRLStreamKit +import IRLStreamKitTestSupport +import Testing + +/// The program-audio takeover contract, pinned on the fake so a consumer's +/// mixer can be built against it without hardware. +/// +/// The rule that matters: while the consumer owns program audio, the built-in +/// microphone is NOT the program source. Everything the viewer hears has to +/// come through `appendProgramAudio` — which is why the mixer must include +/// the mic in its own mix, and why forgetting to start the source leaves the +/// broadcast on bare mic. +@MainActor +struct ProgramAudioTests { + private func silentBuffer() -> CMSampleBuffer? { + var format: CMFormatDescription? + var asbd = AudioStreamBasicDescription( + mSampleRate: 48000, mFormatID: kAudioFormatLinearPCM, + mFormatFlags: kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked, + mBytesPerPacket: 2, mFramesPerPacket: 1, mBytesPerFrame: 2, + mChannelsPerFrame: 1, mBitsPerChannel: 16, mReserved: 0 + ) + CMAudioFormatDescriptionCreate(allocator: kCFAllocatorDefault, + asbd: &asbd, layoutSize: 0, layout: nil, + magicCookieSize: 0, magicCookie: nil, + extensions: nil, formatDescriptionOut: &format) + guard let format else { return nil } + var buffer: CMSampleBuffer? + CMSampleBufferCreate(allocator: kCFAllocatorDefault, dataBuffer: nil, + dataReady: false, makeDataReadyCallback: nil, + refcon: nil, formatDescription: format, + sampleCount: 0, sampleTimingEntryCount: 0, + sampleTimingArray: nil, sampleSizeEntryCount: 0, + sampleSizeArray: nil, sampleBufferOut: &buffer) + return buffer + } + + @Test("Program audio is off until the consumer takes it over") + func inactiveByDefault() { + let fake = FakeStreamEngine() + #expect(!fake.isProgramAudioActive) + } + + @Test("Taking over routes appended buffers; stopping hands the mic back") + func takeoverAndRelease() throws { + let fake = FakeStreamEngine() + let buffer = try #require(silentBuffer()) + + // Buffers pushed before the takeover are dropped rather than queued — + // there is no source to hold them. + fake.appendProgramAudio(buffer) + #expect(fake.programAudioBufferCount == 0) + + fake.startProgramAudio(targetLatency: 0.2) + #expect(fake.isProgramAudioActive) + #expect(fake.programAudioLatency == 0.2) + fake.appendProgramAudio(buffer) + fake.appendProgramAudio(buffer) + #expect(fake.programAudioBufferCount == 2) + + fake.stopProgramAudio() + #expect(!fake.isProgramAudioActive) + fake.appendProgramAudio(buffer) + #expect(fake.programAudioBufferCount == 2) // dropped again + } + + @Test("Taking over twice is idempotent, not a second source") + func takeoverIsIdempotent() { + let fake = FakeStreamEngine() + + fake.startProgramAudio(targetLatency: 0.2) + fake.startProgramAudio(targetLatency: 0.9) + + #expect(fake.programAudioLatency == 0.2) + } + + @Test("The default latency is the documented 200 ms") + func defaultLatency() { + let fake = FakeStreamEngine() + + fake.startProgramAudio() + + #expect(fake.programAudioLatency == 0.2) + } +}