From 53a998c7eb75cd1e339188adbf23466d7d04005e Mon Sep 17 00:00:00 2001 From: TN019 Date: Fri, 24 Jul 2026 10:11:19 +1000 Subject: [PATCH] feat: meeting recordings capture system audio via Core Audio process tap (macOS 15+) Co-Authored-By: Claude Fable 5 --- .../zh-Hans.lproj/Localizable.strings | 4 +- ShiftlyApp/Sources/ShiftlyApp/AppModel.swift | 48 ++++- .../Sources/ShiftlyApp/MeetingViews.swift | 2 +- .../ShiftlyApp/SystemAudioRecorder.swift | 172 ++++++++++++++++++ ShiftlyApp/Sources/ShiftlyKit/Meetings.swift | 5 +- .../Tests/ShiftlyKitTests/MeetingsTests.swift | 4 + docs/SETUP.md | 2 +- scripts/build_app.sh | 2 + 8 files changed, 232 insertions(+), 7 deletions(-) create mode 100644 ShiftlyApp/Sources/ShiftlyApp/SystemAudioRecorder.swift diff --git a/ShiftlyApp/Localization/zh-Hans.lproj/Localizable.strings b/ShiftlyApp/Localization/zh-Hans.lproj/Localizable.strings index 4d811d4..667949c 100644 --- a/ShiftlyApp/Localization/zh-Hans.lproj/Localizable.strings +++ b/ShiftlyApp/Localization/zh-Hans.lproj/Localizable.strings @@ -447,7 +447,9 @@ "Meetings" = "会议"; "Record a meeting" = "录制会议"; "Recording…" = "录音中…"; -"Audio lands in a timestamped folder; Scripto adds the transcript next to it." = "音频存入按时间戳命名的文件夹；Scripto 的转录文件会生成在旁边。"; +"Records your mic plus the Mac's system audio (calls, meetings); Scripto adds the transcript next to it." = "同时录制麦克风和 Mac 系统声音（通话、会议）；Scripto 的转录文件会生成在旁边。"; +"Recording… (system audio unavailable, mic only)" = "录音中…（系统声音不可用，仅麦克风）"; +"Saving recording…" = "正在保存录音…"; "No meetings yet — hit record." = "还没有会议——点击录音开始。"; "Transcribe" = "转录"; "Translate" = "翻译"; diff --git a/ShiftlyApp/Sources/ShiftlyApp/AppModel.swift b/ShiftlyApp/Sources/ShiftlyApp/AppModel.swift index 7dd7a25..500c621 100644 --- a/ShiftlyApp/Sources/ShiftlyApp/AppModel.swift +++ b/ShiftlyApp/Sources/ShiftlyApp/AppModel.swift @@ -90,6 +90,9 @@ final class AppModel: ObservableObject { /// Meeting folders with a Scripto run in flight. @Published var scriptoBusy: Set = [] private var audioRecorder: AVAudioRecorder? + /// SystemAudioRecorder on macOS 15+; typed AnyObject so the stored + /// property compiles against the macOS 13 deployment target. + private var systemAudio: AnyObject? private var recordTimer: Timer? struct ImportCalendar: Identifiable, Equatable { @@ -539,6 +542,24 @@ final class AppModel: ObservableObject { return } audioRecorder = recorder + // System audio side-track (macOS 15+): captures the far side of + // online meetings even over headphones; mixed in on stop. Any + // failure (permission declined, tap error) degrades to mic-only. + var systemAudioOK = false + if #available(macOS 15.0, *) { + let sysRecorder = SystemAudioRecorder() + do { + try sysRecorder.start( + writingTo: SystemAudioRecorder.systemTrackURL( + forMic: URL(fileURLWithPath: path)) + ) + systemAudio = sysRecorder + systemAudioOK = true + } catch { + sysRecorder.stop() + systemAudio = nil + } + } isRecording = true refreshNextShift() // widget snapshot picks up the recording state recordingSeconds = 0 @@ -548,7 +569,9 @@ final class AppModel: ObservableObject { self.recordingSeconds += 1 } } - statusMessage = L("Recording…") + statusMessage = systemAudioOK + ? L("Recording…") + : L("Recording… (system audio unavailable, mic only)") } catch { statusMessage = LF("Could not start recording: %@", error.localizedDescription) } @@ -556,14 +579,33 @@ final class AppModel: ObservableObject { func stopRecording() { guard isRecording else { return } + let micURL = audioRecorder?.url audioRecorder?.stop() audioRecorder = nil recordTimer?.invalidate() recordTimer = nil isRecording = false - refreshMeetings() refreshNextShift() // widget snapshot drops the recording state - statusMessage = L("Recording saved.") + if #available(macOS 15.0, *), + let sysRecorder = systemAudio as? SystemAudioRecorder, let micURL { + sysRecorder.stop() + systemAudio = nil + // The meeting appears in the list only once the mix is done, so + // a transcription can never race against the file swap. + statusMessage = L("Saving recording…") + Task { @MainActor in + _ = await SystemAudioRecorder.mix( + mic: micURL, + system: SystemAudioRecorder.systemTrackURL(forMic: micURL) + ) + refreshMeetings() + statusMessage = L("Recording saved.") + } + } else { + systemAudio = nil + refreshMeetings() + statusMessage = L("Recording saved.") + } } /// Run Scripto headlessly on a meeting's audio; the SRT lands next to diff --git a/ShiftlyApp/Sources/ShiftlyApp/MeetingViews.swift b/ShiftlyApp/Sources/ShiftlyApp/MeetingViews.swift index 632e127..1758a0b 100644 --- a/ShiftlyApp/Sources/ShiftlyApp/MeetingViews.swift +++ b/ShiftlyApp/Sources/ShiftlyApp/MeetingViews.swift @@ -35,7 +35,7 @@ extension ContentView { .monospacedDigit() .foregroundStyle(.red) } else { - Text("Audio lands in a timestamped folder; Scripto adds the transcript next to it.") + Text("Records your mic plus the Mac's system audio (calls, meetings); Scripto adds the transcript next to it.") .font(.caption) .foregroundStyle(.secondary) } diff --git a/ShiftlyApp/Sources/ShiftlyApp/SystemAudioRecorder.swift b/ShiftlyApp/Sources/ShiftlyApp/SystemAudioRecorder.swift new file mode 100644 index 0000000..831feee --- /dev/null +++ b/ShiftlyApp/Sources/ShiftlyApp/SystemAudioRecorder.swift @@ -0,0 +1,172 @@ +import AVFoundation +import CoreAudio +import Foundation + +/// Records the Mac's system audio output — what DingTalk / Zoom / Tencent +/// Meeting play — via a Core Audio process tap. The tap sits on the mixed +/// output of every other process, so the far side of a call is captured +/// even when it only reaches headphones and never touches the microphone. +/// +/// First use triggers the "System Audio Recording" privacy prompt +/// (NSAudioCaptureUsageDescription); if the user declines, `start` throws +/// and the caller falls back to microphone-only recording. +@available(macOS 15.0, *) +final class SystemAudioRecorder { + private var tapID = AudioObjectID(kAudioObjectUnknown) + private var aggregateID = AudioObjectID(kAudioObjectUnknown) + private var ioProcID: AudioDeviceIOProcID? + private var file: AVAudioFile? + private let queue = DispatchQueue(label: "com.shiftly.system-audio") + + struct TapError: LocalizedError { + let stage: String + let status: OSStatus + var errorDescription: String? { "\(stage) (OSStatus \(status))" } + } + + /// Side-track written next to the mic recording while both run; + /// hidden so a crash never leaves a visible stray, and `MeetingStore` + /// skips dotfiles when picking a meeting's audio. + static func systemTrackURL(forMic url: URL) -> URL { + url.deletingLastPathComponent().appendingPathComponent( + "." + url.deletingPathExtension().lastPathComponent + ".system.m4a" + ) + } + + func start(writingTo url: URL) throws { + // Global stereo mixdown of every process except Shiftly itself + // (excluding ourselves avoids feeding notification sounds back in). + let description = CATapDescription(stereoGlobalTapButExcludeProcesses: []) + description.uuid = UUID() + description.muteBehavior = .unmuted + description.isPrivate = true + var newTapID = AudioObjectID(kAudioObjectUnknown) + var status = AudioHardwareCreateProcessTap(description, &newTapID) + guard status == noErr, newTapID != kAudioObjectUnknown else { + throw TapError(stage: "create tap", status: status) + } + tapID = newTapID + + var address = AudioObjectPropertyAddress( + mSelector: kAudioTapPropertyFormat, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + var asbd = AudioStreamBasicDescription() + var size = UInt32(MemoryLayout.size) + status = AudioObjectGetPropertyData(tapID, &address, 0, nil, &size, &asbd) + guard status == noErr, let format = AVAudioFormat(streamDescription: &asbd) else { + stop() + throw TapError(stage: "read tap format", status: status) + } + + // A private aggregate device whose only member is the tap gives us + // a normal IOProc callback carrying the tapped buffers. + let aggregate: [String: Any] = [ + kAudioAggregateDeviceNameKey: "Shiftly System Audio", + kAudioAggregateDeviceUIDKey: UUID().uuidString, + kAudioAggregateDeviceIsPrivateKey: true, + kAudioAggregateDeviceIsStackedKey: false, + kAudioAggregateDeviceTapAutoStartKey: true, + kAudioAggregateDeviceSubDeviceListKey: [[String: Any]](), + kAudioAggregateDeviceTapListKey: [ + [ + kAudioSubTapUIDKey: description.uuid.uuidString, + kAudioSubTapDriftCompensationKey: true, + ] + ], + ] + status = AudioHardwareCreateAggregateDevice(aggregate as CFDictionary, &aggregateID) + guard status == noErr else { + stop() + throw TapError(stage: "create aggregate device", status: status) + } + + let file = try AVAudioFile( + forWriting: url, + settings: [ + AVFormatIDKey: kAudioFormatMPEG4AAC, + AVSampleRateKey: asbd.mSampleRate, + AVNumberOfChannelsKey: asbd.mChannelsPerFrame, + AVEncoderAudioQualityKey: AVAudioQuality.high.rawValue, + ], + commonFormat: format.commonFormat, + interleaved: format.isInterleaved + ) + self.file = file + status = AudioDeviceCreateIOProcIDWithBlock(&ioProcID, aggregateID, queue) { + _, inInputData, _, _, _ in + guard let buffer = AVAudioPCMBuffer( + pcmFormat: format, bufferListNoCopy: inInputData, deallocator: nil + ) else { return } + try? file.write(from: buffer) + } + guard status == noErr, ioProcID != nil else { + stop() + throw TapError(stage: "create IO proc", status: status) + } + status = AudioDeviceStart(aggregateID, ioProcID) + guard status == noErr else { + stop() + throw TapError(stage: "start device", status: status) + } + } + + /// Stops and tears down; safe to call at any point, including from a + /// failed `start`. + func stop() { + if let ioProcID, aggregateID != kAudioObjectUnknown { + AudioDeviceStop(aggregateID, ioProcID) + AudioDeviceDestroyIOProcID(aggregateID, ioProcID) + } + ioProcID = nil + if aggregateID != kAudioObjectUnknown { + AudioHardwareDestroyAggregateDevice(aggregateID) + aggregateID = kAudioObjectUnknown + } + if tapID != kAudioObjectUnknown { + AudioHardwareDestroyProcessTap(tapID) + tapID = kAudioObjectUnknown + } + file = nil + } + + /// Mixes the hidden system track into the mic recording (one m4a under + /// the mic file's name) and removes the side-track. Returns false when + /// there was nothing usable to mix — the mic file stays as recorded. + static func mix(mic: URL, system: URL) async -> Bool { + defer { try? FileManager.default.removeItem(at: system) } + guard FileManager.default.fileExists(atPath: system.path) else { return false } + let composition = AVMutableComposition() + var added = 0 + for url in [mic, system] { + let asset = AVURLAsset(url: url) + guard let track = try? await asset.loadTracks(withMediaType: .audio).first, + let duration = try? await asset.load(.duration), + duration > .zero, + let target = composition.addMutableTrack( + withMediaType: .audio, preferredTrackID: kCMPersistentTrackID_Invalid + ), + (try? target.insertTimeRange( + CMTimeRange(start: .zero, duration: duration), of: track, at: .zero + )) != nil + else { continue } + added += 1 + } + guard added == 2, + let session = AVAssetExportSession( + asset: composition, presetName: AVAssetExportPresetAppleM4A + ) + else { return false } + let scratch = mic.deletingLastPathComponent().appendingPathComponent(".mixing.m4a") + try? FileManager.default.removeItem(at: scratch) + do { + try await session.export(to: scratch, as: .m4a) + _ = try FileManager.default.replaceItemAt(mic, withItemAt: scratch) + return true + } catch { + try? FileManager.default.removeItem(at: scratch) + return false + } + } +} diff --git a/ShiftlyApp/Sources/ShiftlyKit/Meetings.swift b/ShiftlyApp/Sources/ShiftlyKit/Meetings.swift index ff55ca4..10bb96b 100644 --- a/ShiftlyApp/Sources/ShiftlyKit/Meetings.swift +++ b/ShiftlyApp/Sources/ShiftlyKit/Meetings.swift @@ -77,7 +77,10 @@ public struct MeetingStore { return names.compactMap { name -> Meeting? in guard let (date, time) = Self.parseFolderName(name) else { return nil } let folder = "\(rootDir)/\(name)" - let files = (try? fm.contentsOfDirectory(atPath: folder)) ?? [] + // Dotfiles are recording scratch (hidden system-audio track, + // in-flight mix) — never a meeting's audio or subtitles. + let files = ((try? fm.contentsOfDirectory(atPath: folder)) ?? []) + .filter { !$0.hasPrefix(".") } let audio = files.first { $0.hasSuffix(".mp4") || $0.hasSuffix(".m4a") } var subtitles: [String: String] = [:] for file in files where file.hasSuffix(".srt") { diff --git a/ShiftlyApp/Tests/ShiftlyKitTests/MeetingsTests.swift b/ShiftlyApp/Tests/ShiftlyKitTests/MeetingsTests.swift index 593a9a3..45c57e7 100644 --- a/ShiftlyApp/Tests/ShiftlyKitTests/MeetingsTests.swift +++ b/ShiftlyApp/Tests/ShiftlyKitTests/MeetingsTests.swift @@ -25,6 +25,10 @@ import Testing let folder = (audio as NSString).deletingLastPathComponent FileManager.default.createFile(atPath: folder + "/19-07-26.en.srt", contents: Data("x".utf8)) FileManager.default.createFile(atPath: folder + "/19-07-26.zh.srt", contents: Data("y".utf8)) + // Recording scratch (hidden system-audio track) must never be + // picked up as the meeting's audio or subtitles. + FileManager.default.createFile(atPath: folder + "/.19-07-26.system.m4a", contents: Data([0])) + FileManager.default.createFile(atPath: folder + "/.junk.srt", contents: Data("z".utf8)) _ = try store.newRecordingPath(date: "2026-07-20", timeHHMM: "09:05") try FileManager.default.createDirectory( atPath: root + "/not a meeting", withIntermediateDirectories: true diff --git a/docs/SETUP.md b/docs/SETUP.md index 8109a1d..c5af6ef 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -66,7 +66,7 @@ The Python helper scripts are bundled into the app, so no repo checkout is neede **Desktop widgets:** `build_app.sh` also compiles a WidgetKit extension into `Contents/PlugIns/ShiftlyWidgets.appex` with plain `swiftc` — no Xcode. Three things make a hand-built appex loadable: the **`com.apple.security.app-sandbox` entitlement** (WidgetKit refuses unsandboxed extensions), `CFBundleSupportedPlatforms = [MacOSX]`, and linking with **`-e _NSExtensionMain`** (a plain Swift `@main` entry dies with "Unrecognized extension type"). The app feeds the widgets by writing a snapshot JSON into the `group.com.shiftly.app` container and calling `WidgetCenter.reloadAllTimelines()`. Add the widgets via right-click on the desktop → Edit Widgets → search "Shiftly"; the chips deep-link back through the `shiftly://` URL scheme (`start-work`, `meetings`, `new-note`, `open`). -**Meetings / Scripto:** recordings land in `/dd-mm-yy | hh-mm/dd-mm-yy.mp4` (AAC). Transcribe / Translate run [Scripto](https://github.com/TN019/scripto) headlessly via `uv run scripto-cli run