diff --git a/Sources/IRLStreamKit/Facade/CameraPreview.swift b/Sources/IRLStreamKit/Facade/CameraPreview.swift index 99e4832..704a343 100644 --- a/Sources/IRLStreamKit/Facade/CameraPreview.swift +++ b/Sources/IRLStreamKit/Facade/CameraPreview.swift @@ -11,6 +11,8 @@ public protocol CameraPreviewSource: AnyObject { func makePreviewUIView() -> UIView } +extension IRLStreamEngine: ProgramAudioSink {} + extension IRLStreamEngine: CameraPreviewSource { public func makePreviewUIView() -> UIView { internalPreviewView diff --git a/Sources/IRLStreamKit/Facade/IRLStreamEngine.swift b/Sources/IRLStreamKit/Facade/IRLStreamEngine.swift index c3d6e23..e53095e 100644 --- a/Sources/IRLStreamKit/Facade/IRLStreamEngine.swift +++ b/Sources/IRLStreamKit/Facade/IRLStreamEngine.swift @@ -4,6 +4,7 @@ // service. import AVFoundation +import CoreMedia import Foundation import IRLTPBonding import Observation @@ -36,6 +37,10 @@ public final class IRLStreamEngine: StreamEngine { @ObservationIgnored private var desiredZoomX: [CameraSelection: Float] = [:] // Moblin's default zoom ramp speed (database.zoom.speed midpoint). private static let zoomRampRate: Float = 5.0 + // App-supplied program audio, if the consumer took it over. Like mute and + // the camera attach, this is lost whenever setNetStream rebuilds the + // Processor, so it is re-established after every rebuild. + @ObservationIgnored private var programAudio: (id: UUID, targetLatency: TimeInterval)? // streamTotal() resets on every (re)connect; accumulate across reconnects. @ObservationIgnored private var totalBytesBase: Int64 = 0 @ObservationIgnored let internalPreviewView: PreviewView @@ -309,6 +314,32 @@ public final class IRLStreamEngine: StreamEngine { device.position == .front ? .front : .back } + // MARK: - Program audio + + public var isProgramAudioActive: Bool { programAudio != nil } + + public func startProgramAudio(targetLatency: TimeInterval) { + guard programAudio == nil else { return } + let id = UUID() + programAudio = (id, targetLatency) + media.addBufferedAudio(cameraId: id, name: "Program", latency: targetLatency) + media.attachBufferedAudio(cameraId: id) + } + + public func appendProgramAudio(_ sampleBuffer: CMSampleBuffer) { + guard let programAudio else { return } + media.appendBufferedAudioSampleBuffer(cameraId: programAudio.id, sampleBuffer: sampleBuffer) + } + + public func stopProgramAudio() { + guard let programAudio else { return } + self.programAudio = nil + // Back to the microphone. `attachBufferedAudio(cameraId: nil)` would + // attach NOTHING (device nil, buffer nil) and leave the stream silent. + media.attachDefaultAudioDevice(builtinDelay: 0) + media.removeBufferedAudio(cameraId: programAudio.id) + } + // MARK: - Signal handling (single consumer preserves callback ordering) private func handle(_ stamped: StampedSignal) { @@ -411,6 +442,14 @@ public final class IRLStreamEngine: StreamEngine { // The fresh Processor lost the mute flag (AudioUnit.muted defaults to // false) — re-apply the user's intent, mirroring Moblin's updateMute(). media.setMute(on: desiredMicMuted) + // setNetStream re-attached the built-in mic; if the app owns program + // audio, take it back or the broadcast silently reverts to bare mic + // at go-live — exactly the class of bug the mute re-apply exists for. + if let programAudio { + media.addBufferedAudio(cameraId: programAudio.id, name: "Program", + latency: programAudio.targetLatency) + media.attachBufferedAudio(cameraId: programAudio.id) + } // Rebind the preview drawable and start the new Processor (mirrors // ModelStream.attachStream). if let processor = media.getProcessor() { diff --git a/Sources/IRLStreamKit/Facade/ProgramAudio.swift b/Sources/IRLStreamKit/Facade/ProgramAudio.swift new file mode 100644 index 0000000..c6af292 --- /dev/null +++ b/Sources/IRLStreamKit/Facade/ProgramAudio.swift @@ -0,0 +1,48 @@ +import CoreMedia +import Foundation + +/// Program audio supplied by the CONSUMER instead of the built-in microphone. +/// +/// Why this exists: the vendored audio path *selects* a source, it never +/// mixes. `AudioUnit.captureOutput` takes one buffer and hands it straight to +/// the encoder, and selecting a buffered source makes the microphone path +/// bail out entirely. So "the streamer's voice **and** the alert/TTS audio" +/// cannot be produced inside the engine at all — there is nowhere for a +/// second stream to join. +/// +/// Rather than fork the vendor to add a mixing stage, this hands the job to +/// the app, which is where the other audio already lives (speech synthesis, +/// alert sounds, the captions tap). The app owns one audio graph, mixes what +/// belongs on the broadcast, and pushes the result here; the engine treats it +/// as the program source and encodes it. +/// +/// Deliberately NOT part of `StreamEngine`: it trades in `CMSampleBuffer`, +/// and the main protocol is kept free of media types so a fake needs zero +/// hardware. Same reasoning as `CameraPreviewSource`. +@MainActor +public protocol ProgramAudioSink: AnyObject { + /// Whether app-supplied audio is currently the program source. + var isProgramAudioActive: Bool { get } + + /// Take over program audio. The built-in microphone stops being the + /// program source — everything the viewer hears now comes from + /// `appendProgramAudio`, so the caller must include the mic in its mix. + /// + /// `targetLatency` is the jitter buffer the engine keeps for this source; + /// it trades delay for tolerance of an irregular feed. + func startProgramAudio(targetLatency: TimeInterval) + + /// Feed one buffer of mixed program audio. + func appendProgramAudio(_ sampleBuffer: CMSampleBuffer) + + /// Hand program audio back to the built-in microphone. + func stopProgramAudio() +} + +public extension ProgramAudioSink { + /// 200 ms: enough to absorb a synthesis burst without a perceptible lag + /// between what the streamer says and what the viewer hears. + func startProgramAudio() { + startProgramAudio(targetLatency: 0.2) + } +} diff --git a/Sources/IRLStreamKitTestSupport/FakeStreamEngine.swift b/Sources/IRLStreamKitTestSupport/FakeStreamEngine.swift index 12d8f8b..821f244 100644 --- a/Sources/IRLStreamKitTestSupport/FakeStreamEngine.swift +++ b/Sources/IRLStreamKitTestSupport/FakeStreamEngine.swift @@ -3,6 +3,7 @@ // engine and enforces the same phase machine, so fake and real cannot // diverge on state derivation or lifecycle contract. +import CoreMedia import Foundation import IRLStreamKit import Observation @@ -34,6 +35,11 @@ public final class FakeStreamEngine: StreamEngine { @ObservationIgnored private let broadcaster = EventBroadcaster() + /// Set while the consumer owns program audio; nil means the built-in mic. + public internal(set) var programAudioLatency: TimeInterval? + /// How many mixed buffers the consumer pushed while it owned program audio. + public internal(set) var programAudioBufferCount = 0 + public init() {} /// Tests drive the world: applies the same package reducer synchronously @@ -162,3 +168,23 @@ public final class FakeStreamEngine: StreamEngine { emit(.stabilizationChanged(mode)) } } + +/// Program-audio conformance for the fake: records the takeover and every +/// buffer count, so a consumer's mixer can be tested with zero hardware. +extension FakeStreamEngine: ProgramAudioSink { + public var isProgramAudioActive: Bool { programAudioLatency != nil } + + public func startProgramAudio(targetLatency: TimeInterval) { + guard programAudioLatency == nil else { return } + programAudioLatency = targetLatency + } + + public func appendProgramAudio(_: CMSampleBuffer) { + guard programAudioLatency != nil else { return } + programAudioBufferCount += 1 + } + + public func stopProgramAudio() { + programAudioLatency = nil + } +} diff --git a/Tests/IRLStreamKitTests/ProgramAudioTests.swift b/Tests/IRLStreamKitTests/ProgramAudioTests.swift new file mode 100644 index 0000000..6d9e9c7 --- /dev/null +++ b/Tests/IRLStreamKitTests/ProgramAudioTests.swift @@ -0,0 +1,87 @@ +import CoreMedia +import Foundation +import IRLStreamKit +import IRLStreamKitTestSupport +import Testing + +/// The program-audio takeover contract, pinned on the fake so a consumer's +/// mixer can be built against it without hardware. +/// +/// The rule that matters: while the consumer owns program audio, the built-in +/// microphone is NOT the program source. Everything the viewer hears has to +/// come through `appendProgramAudio` — which is why the mixer must include +/// the mic in its own mix, and why forgetting to start the source leaves the +/// broadcast on bare mic. +@MainActor +struct ProgramAudioTests { + private func silentBuffer() -> CMSampleBuffer? { + var format: CMFormatDescription? + var asbd = AudioStreamBasicDescription( + mSampleRate: 48000, mFormatID: kAudioFormatLinearPCM, + mFormatFlags: kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked, + mBytesPerPacket: 2, mFramesPerPacket: 1, mBytesPerFrame: 2, + mChannelsPerFrame: 1, mBitsPerChannel: 16, mReserved: 0 + ) + CMAudioFormatDescriptionCreate(allocator: kCFAllocatorDefault, + asbd: &asbd, layoutSize: 0, layout: nil, + magicCookieSize: 0, magicCookie: nil, + extensions: nil, formatDescriptionOut: &format) + guard let format else { return nil } + var buffer: CMSampleBuffer? + CMSampleBufferCreate(allocator: kCFAllocatorDefault, dataBuffer: nil, + dataReady: false, makeDataReadyCallback: nil, + refcon: nil, formatDescription: format, + sampleCount: 0, sampleTimingEntryCount: 0, + sampleTimingArray: nil, sampleSizeEntryCount: 0, + sampleSizeArray: nil, sampleBufferOut: &buffer) + return buffer + } + + @Test("Program audio is off until the consumer takes it over") + func inactiveByDefault() { + let fake = FakeStreamEngine() + #expect(!fake.isProgramAudioActive) + } + + @Test("Taking over routes appended buffers; stopping hands the mic back") + func takeoverAndRelease() throws { + let fake = FakeStreamEngine() + let buffer = try #require(silentBuffer()) + + // Buffers pushed before the takeover are dropped rather than queued — + // there is no source to hold them. + fake.appendProgramAudio(buffer) + #expect(fake.programAudioBufferCount == 0) + + fake.startProgramAudio(targetLatency: 0.2) + #expect(fake.isProgramAudioActive) + #expect(fake.programAudioLatency == 0.2) + fake.appendProgramAudio(buffer) + fake.appendProgramAudio(buffer) + #expect(fake.programAudioBufferCount == 2) + + fake.stopProgramAudio() + #expect(!fake.isProgramAudioActive) + fake.appendProgramAudio(buffer) + #expect(fake.programAudioBufferCount == 2) // dropped again + } + + @Test("Taking over twice is idempotent, not a second source") + func takeoverIsIdempotent() { + let fake = FakeStreamEngine() + + fake.startProgramAudio(targetLatency: 0.2) + fake.startProgramAudio(targetLatency: 0.9) + + #expect(fake.programAudioLatency == 0.2) + } + + @Test("The default latency is the documented 200 ms") + func defaultLatency() { + let fake = FakeStreamEngine() + + fake.startProgramAudio() + + #expect(fake.programAudioLatency == 0.2) + } +}