From 57166742c9f6aed22bfdda756dc8af0f3be008ae Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 22:37:54 +0000 Subject: [PATCH 1/3] Paste into browsers even when AX exposes no editable signal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Web content is routinely invisible to Accessibility — Chromium builds its tree lazily (the first query resolves only a bare AXWebArea), and contenteditable composers like ChatGPT's expose no editable role even with the tree live — so the pre-paste editability gate read real browser text fields as non-editable and degraded dictations to copy-only. Extend the AX-opaque exemption that already covered Electron editors to known browsers (matched by bundle-identifier prefix, covering channel variants): FocusCapture.isAXOpaqueApp = isElectronApp || isBrowserApp now backs KeyInjector's isAXOpaqueEditor seam. The accepted trade-off is a rare beep when a browser truly has nothing editable focused, instead of dropping the user's words to the clipboard. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014iyxjKUdUqKdDXzj24opJt --- BLURTENGINE.md | 2 +- .../FocusCapture+Editability.swift | 69 +++++++++++++++---- .../BlurtEngine/Injection/KeyInjector.swift | 21 +++--- .../BrowserBundleIDTests.swift | 62 +++++++++++++++++ .../EditableTargetTests.swift | 5 +- 5 files changed, 135 insertions(+), 24 deletions(-) create mode 100644 Tests/BlurtEngineTests/BrowserBundleIDTests.swift diff --git a/BLURTENGINE.md b/BLURTENGINE.md index d0c2f9c..5332f98 100644 --- a/BLURTENGINE.md +++ b/BLURTENGINE.md @@ -144,7 +144,7 @@ func setTargetApp(_ app: NSRunningApplication?) async func insert(_ text: String, after priorText: String?, windowTitle: String?) async throws ``` -`KeyInjector.insert` **always** pastes: it saves the current pasteboard, writes the transcript, activates the captured target app, posts a synthesized ⌘V, waits for the target to read the clipboard (`pasteSettleDuration`, default 400 ms, tunable in the initializer), then restores the prior pasteboard contents. That restore never destroys what it can't put back: an unreadable pasteboard snapshots as `nil` (not as empty) and is skipped, and the replacement items are built before `clearContents()`, with the plain-string flavor as a floor when promised representations can't be materialized. There is no keystroke-by-keystroke typing path and no length threshold. If the target app is gone or nothing editable is focused it leaves the text on the clipboard and throws `.targetAppLost` / `.noEditableTarget` — which the session turns into the quiet `.noTarget` outcome. `priorText` (the text before the caret, captured at press time) drives `withLeadingSeparator`, which joins consecutive dictations with a space so they don't run together. When `priorText` is unreadable (an Accessibility-opaque editor, or a browser tab like Google Docs whose canvas-rendered body exposes no AX text), `separatorBasis` falls back to what was last pasted — but only when both the target app **and** `windowTitle` match the last successful insert, so the fallback tracks "the same window," not just "the same process" (a browser hosts many unrelated tabs/documents under one PID). +`KeyInjector.insert` **always** pastes: it saves the current pasteboard, writes the transcript, activates the captured target app, posts a synthesized ⌘V, waits for the target to read the clipboard (`pasteSettleDuration`, default 400 ms, tunable in the initializer), then restores the prior pasteboard contents. That restore never destroys what it can't put back: an unreadable pasteboard snapshots as `nil` (not as empty) and is skipped, and the replacement items are built before `clearContents()`, with the plain-string flavor as a floor when promised representations can't be materialized. There is no keystroke-by-keystroke typing path and no length threshold. If the target app is gone or nothing editable is focused it leaves the text on the clipboard and throws `.targetAppLost` / `.noEditableTarget` — which the session turns into the quiet `.noTarget` outcome. AX-opaque targets — Electron editors and web browsers, per `FocusCapture.isAXOpaqueApp` — are exempt from the editability gate and are pasted into even with no editable AX signal: web content is routinely invisible to Accessibility (Chromium builds its tree lazily; `contenteditable` composers expose no editable role), so "no signal" there usually means "AX can't see the field," not "no field," and the accepted trade-off is a rare beep over dropping the user's words to copy-only. `priorText` (the text before the caret, captured at press time) drives `withLeadingSeparator`, which joins consecutive dictations with a space so they don't run together. When `priorText` is unreadable (an Accessibility-opaque editor, or a browser tab like Google Docs whose canvas-rendered body exposes no AX text), `separatorBasis` falls back to what was last pasted — but only when both the target app **and** `windowTitle` match the last successful insert, so the fallback tracks "the same window," not just "the same process" (a browser hosts many unrelated tabs/documents under one PID). The session calls `setTargetApp` at press time with the app that was frontmost when recording started — so the paste lands where the user was, even if focus moved during transcription. diff --git a/Sources/BlurtEngine/FocusCapture/FocusCapture+Editability.swift b/Sources/BlurtEngine/FocusCapture/FocusCapture+Editability.swift index 3af4b79..0e9257d 100644 --- a/Sources/BlurtEngine/FocusCapture/FocusCapture+Editability.swift +++ b/Sources/BlurtEngine/FocusCapture/FocusCapture+Editability.swift @@ -28,11 +28,11 @@ extension FocusCapture { /// no readable role — is treated as not editable, so we copy rather than beep a /// ⌘V into a target that can't take it. /// - /// AX-opaque Electron editors (VS Code, Slack) expose *none* of these signals - /// even for a genuine text field, so this returns false for them too — but the - /// injector still pastes into those via a separate Electron-app check (see - /// `isElectronApp` / `KeyInjector.insert`), so the user's words aren't dropped - /// to copy-only there. + /// AX-opaque apps — Electron editors (VS Code, Slack) and web browsers — can + /// expose *none* of these signals even for a genuine text field, so this + /// returns false for them too. The injector still pastes into those via a + /// separate app-identity check (see `isAXOpaqueApp` / `KeyInjector.insert`), + /// so the user's words aren't dropped to copy-only there. static func isEditableTarget(role: String?, valueSettable: Bool, hasInsertionPoint: Bool) -> Bool { if let role, editableRoles.contains(role) { return true } return valueSettable || hasInsertionPoint @@ -41,10 +41,9 @@ extension FocusCapture { /// Whether `app` is an Electron/Chromium-based app, detected by the bundled /// Electron framework. Such apps ship with their accessibility tree off, so even /// a focused text field exposes no editable AX signal and - /// `hasEditableFocusedElement` reads them as non-editable. They're the one case - /// the injector still pastes into on no signal (dropping the user's words into a - /// copy-only fallback would be the worse mistake). A native app with genuinely no - /// editable focus bundles no such framework and correctly falls back to copy. + /// `hasEditableFocusedElement` reads them as non-editable. A native app with + /// genuinely no editable focus bundles no such framework and correctly falls + /// back to copy. static func isElectronApp(_ app: NSRunningApplication?) -> Bool { guard let bundleURL = app?.bundleURL else { return false } let electronFramework = bundleURL.appendingPathComponent( @@ -52,6 +51,51 @@ extension FocusCapture { return FileManager.default.fileExists(atPath: electronFramework.path) } + /// Bundle-identifier prefixes of known web browsers. Prefix-matched so channel + /// variants classify with their stable siblings (`com.google.Chrome.beta`, + /// `com.apple.SafariTechnologyPreview`). + private static let browserBundleIDPrefixes: [String] = [ + "com.apple.Safari", // Safari + Safari Technology Preview + "com.google.Chrome", // Chrome + Beta/Dev/Canary + "org.chromium.Chromium", + "com.microsoft.edgemac", // Edge + Beta/Dev/Canary + "com.brave.Browser", // Brave + Beta/Nightly + "com.operasoftware.Opera", + "com.vivaldi.Vivaldi", + "company.thebrowser.Browser", // Arc + "org.mozilla.firefox", + "com.duckduckgo.macos.browser", + "com.kagi.kagimacOS", // Orion + ] + + /// Pure decision behind `isBrowserApp`: does this bundle identifier belong to a + /// known browser? Split from the `NSRunningApplication` wrapper so the + /// classification is unit-testable without live running apps. + static func isBrowserBundleID(_ bundleID: String?) -> Bool { + guard let bundleID else { return false } + return browserBundleIDPrefixes.contains { bundleID.hasPrefix($0) } + } + + /// Whether `app` is a known web browser. Web content is AX-opaque in practice: + /// Chromium builds its accessibility tree lazily (the first query after launch + /// resolves only a bare `AXWebArea` with no editable signal), and even with the + /// tree live, a `contenteditable` composer (ChatGPT's ProseMirror field) can + /// surface as a generic group with no settable value. So "no editable signal" + /// in a browser usually means "AX can't see the field," not "no field." + static func isBrowserApp(_ app: NSRunningApplication?) -> Bool { + isBrowserBundleID(app?.bundleIdentifier) + } + + /// Whether `app` is AX-opaque — an Electron editor or a web browser — where a + /// focused text field can expose no editable AX signal at all. These are the + /// one case the injector still pastes into on no signal: dropping the user's + /// words into a copy-only fallback there would be the worse mistake. The + /// accepted trade-off is a rare ⌘V beep when such an app truly has nothing + /// editable focused. + static func isAXOpaqueApp(_ app: NSRunningApplication?) -> Bool { + isElectronApp(app) || isBrowserApp(app) + } + /// Whether the system-wide focused element can accept pasted text right now. /// Read by `KeyInjector` (off the main actor, after it has activated the target /// app) just before pasting — the Accessibility *client* read APIs are @@ -66,9 +110,10 @@ extension FocusCapture { // AX is trusted but reports no focused element — e.g. a native app frontmost // with nothing editable focused (Finder, the desktop, a button-only window). // Posting ⌘V there only beeps, so treat it as non-editable and copy instead. - // AX-opaque Electron apps (VS Code, Slack) also expose no focused element - // here, but the injector's Electron-app check still pastes into those (see - // `KeyInjector.insert` / `isElectronApp`). + // AX-opaque apps (Electron editors like VS Code/Slack, and browsers before + // Chromium's lazy accessibility tree is built) also expose no focused + // element here, but the injector's app-identity check still pastes into + // those (see `KeyInjector.insert` / `isAXOpaqueApp`). return false } diff --git a/Sources/BlurtEngine/Injection/KeyInjector.swift b/Sources/BlurtEngine/Injection/KeyInjector.swift index 8eba297..cb20722 100644 --- a/Sources/BlurtEngine/Injection/KeyInjector.swift +++ b/Sources/BlurtEngine/Injection/KeyInjector.swift @@ -88,12 +88,14 @@ public actor KeyInjector: InjectorProtocol { /// host's live focus (defaults to "editable" there). private let hasEditableTarget: @Sendable () -> Bool - /// Whether the captured target app is an AX-opaque Electron/Chromium editor - /// (VS Code, Slack), which exposes no editable AX signal even for a real text - /// field. When `hasEditableTarget` reads false but this is true, we still paste - /// rather than copy — dropping the user's words into an Electron editor they're - /// clearly typing in would be the worse mistake. Injectable so tests don't - /// depend on which apps are installed (defaults to "not Electron"). + /// Whether the captured target app is AX-opaque — an Electron/Chromium editor + /// (VS Code, Slack) or a web browser — which can expose no editable AX signal + /// even for a real text field (Chromium builds its accessibility tree lazily, + /// and `contenteditable` composers like ChatGPT's surface no editable role). + /// When `hasEditableTarget` reads false but this is true, we still paste + /// rather than copy — dropping the user's words into a field they're clearly + /// typing in would be the worse mistake. Injectable so tests don't depend on + /// which apps are installed (defaults to "not opaque"). private let isAXOpaqueEditor: @Sendable (NSRunningApplication?) -> Bool /// The pasteboard the paste reads, writes, and restores. Behind a seam so @@ -111,7 +113,7 @@ public actor KeyInjector: InjectorProtocol { waitForTargetActivation: KeyInjector.waitUntilFrontmost, isAccessibilityTrusted: KeyInjector.accessibilityTrusted, hasEditableTarget: FocusCapture.hasEditableFocusedElement, - isAXOpaqueEditor: FocusCapture.isElectronApp) + isAXOpaqueEditor: FocusCapture.isAXOpaqueApp) } init( @@ -232,8 +234,9 @@ public actor KeyInjector: InjectorProtocol { // a synthesized ⌘V would just make macOS beep. Leave the transcript on the // clipboard so the user can paste it by hand, and signal the pipeline to show // a quiet "copied" notice instead of typing. The exception is an AX-opaque - // Electron editor (VS Code, Slack), which reports no editable signal even for - // a real text field — there we still paste rather than drop the user's words. + // app — an Electron editor (VS Code, Slack) or a web browser — which can + // report no editable signal even for a real text field; there we still paste + // rather than drop the user's words. guard hasEditableTarget() || isAXOpaqueEditor(target) else { clipboard.write(finalText) throw BlurtError.noEditableTarget diff --git a/Tests/BlurtEngineTests/BrowserBundleIDTests.swift b/Tests/BlurtEngineTests/BrowserBundleIDTests.swift new file mode 100644 index 0000000..3073499 --- /dev/null +++ b/Tests/BlurtEngineTests/BrowserBundleIDTests.swift @@ -0,0 +1,62 @@ +import Testing + +@testable import BlurtEngine + +/// Pins the browser classification behind the injector's AX-opaque exemption +/// (see `FocusCapture.isAXOpaqueApp`): a known browser pastes even when the +/// focused element exposes no editable AX signal, because web content is +/// routinely opaque (Chromium's lazy accessibility tree, `contenteditable` +/// composers like ChatGPT's) — "no signal" there means "AX can't see the +/// field," not "no field." +@Suite("FocusCapture.isBrowserBundleID") +struct BrowserBundleIDTests { + @Test( + "known browser bundle IDs classify as browsers", + arguments: [ + "com.apple.Safari", + "com.google.Chrome", + "org.chromium.Chromium", + "com.microsoft.edgemac", + "com.brave.Browser", + "com.operasoftware.Opera", + "com.vivaldi.Vivaldi", + "company.thebrowser.Browser", + "org.mozilla.firefox", + "com.duckduckgo.macos.browser", + "com.kagi.kagimacOS", + ]) + func knownBrowsers(bundleID: String) { + #expect(FocusCapture.isBrowserBundleID(bundleID)) + } + + @Test( + "channel variants classify with their stable siblings (prefix match)", + arguments: [ + "com.apple.SafariTechnologyPreview", + "com.google.Chrome.beta", + "com.google.Chrome.canary", + "com.microsoft.edgemac.Dev", + "com.brave.Browser.nightly", + ]) + func channelVariants(bundleID: String) { + #expect(FocusCapture.isBrowserBundleID(bundleID)) + } + + @Test( + "non-browser apps are not browsers — they keep the copy-don't-beep fallback", + arguments: [ + "com.apple.finder", + "com.apple.TextEdit", + "com.apple.dt.Xcode", + "com.microsoft.VSCode", // Electron: exempted by isElectronApp, not here + "com.googlecode.iterm2", // "com.google" lookalike must not prefix-match + ]) + func nonBrowsers(bundleID: String) { + #expect(!FocusCapture.isBrowserBundleID(bundleID)) + } + + @Test("a nil bundle ID is not a browser") + func nilBundleID() { + #expect(!FocusCapture.isBrowserBundleID(nil)) + } +} diff --git a/Tests/BlurtEngineTests/EditableTargetTests.swift b/Tests/BlurtEngineTests/EditableTargetTests.swift index c61ad81..27e45b7 100644 --- a/Tests/BlurtEngineTests/EditableTargetTests.swift +++ b/Tests/BlurtEngineTests/EditableTargetTests.swift @@ -46,8 +46,9 @@ struct EditableTargetTests { func unknownRoleWithoutSignalCopies() { // A focused element that reports an unrecognized role and exposes no settable // value or insertion point isn't a text target — copy rather than beep a ⌘V - // into it. (AX-opaque Electron editors, which also land here, are pasted into - // via the injector's separate Electron-app check, not this signal test.) + // into it. (AX-opaque apps — Electron editors and browsers — also land here, + // but are pasted into via the injector's separate app-identity check, not + // this signal test.) #expect( !FocusCapture.isEditableTarget( role: "AXWebArea", valueSettable: false, hasInsertionPoint: false)) From c8c6588574810a805664bc3782453170c289a48d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 22:38:44 +0000 Subject: [PATCH 2/3] Check the cheap browser match before the Electron disk probe isAXOpaqueApp now tries the bundle-ID prefix check before isElectronApp's FileManager.fileExists probe, skipping synchronous file I/O on the injector actor for the common browser case. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014iyxjKUdUqKdDXzj24opJt --- .../BlurtEngine/FocusCapture/FocusCapture+Editability.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Sources/BlurtEngine/FocusCapture/FocusCapture+Editability.swift b/Sources/BlurtEngine/FocusCapture/FocusCapture+Editability.swift index 0e9257d..b8ee2a8 100644 --- a/Sources/BlurtEngine/FocusCapture/FocusCapture+Editability.swift +++ b/Sources/BlurtEngine/FocusCapture/FocusCapture+Editability.swift @@ -93,7 +93,9 @@ extension FocusCapture { /// accepted trade-off is a rare ⌘V beep when such an app truly has nothing /// editable focused. static func isAXOpaqueApp(_ app: NSRunningApplication?) -> Bool { - isElectronApp(app) || isBrowserApp(app) + // Browser first: it's a string prefix check, whereas isElectronApp probes + // the disk (FileManager.fileExists) — skip that I/O for the common case. + isBrowserApp(app) || isElectronApp(app) } /// Whether the system-wide focused element can accept pasted text right now. From a719e6298b877068dd013830975440e10b1b6b12 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 01:35:44 +0000 Subject: [PATCH 3/3] Rename the injector's isAXOpaqueEditor seam to isAXOpaqueApp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seam now covers web browsers as well as Electron editors, and its default is FocusCapture.isAXOpaqueApp — align the property, initializer label, and test call site with that name. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014iyxjKUdUqKdDXzj24opJt --- Sources/BlurtEngine/Injection/KeyInjector.swift | 10 +++++----- Tests/BlurtEngineTests/KeyInjectorInsertTests.swift | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Sources/BlurtEngine/Injection/KeyInjector.swift b/Sources/BlurtEngine/Injection/KeyInjector.swift index cb20722..2d0f54d 100644 --- a/Sources/BlurtEngine/Injection/KeyInjector.swift +++ b/Sources/BlurtEngine/Injection/KeyInjector.swift @@ -96,7 +96,7 @@ public actor KeyInjector: InjectorProtocol { /// rather than copy — dropping the user's words into a field they're clearly /// typing in would be the worse mistake. Injectable so tests don't depend on /// which apps are installed (defaults to "not opaque"). - private let isAXOpaqueEditor: @Sendable (NSRunningApplication?) -> Bool + private let isAXOpaqueApp: @Sendable (NSRunningApplication?) -> Bool /// The pasteboard the paste reads, writes, and restores. Behind a seam so /// tests exercise the save/restore + changeCount logic against an in-memory @@ -113,7 +113,7 @@ public actor KeyInjector: InjectorProtocol { waitForTargetActivation: KeyInjector.waitUntilFrontmost, isAccessibilityTrusted: KeyInjector.accessibilityTrusted, hasEditableTarget: FocusCapture.hasEditableFocusedElement, - isAXOpaqueEditor: FocusCapture.isAXOpaqueApp) + isAXOpaqueApp: FocusCapture.isAXOpaqueApp) } init( @@ -123,7 +123,7 @@ public actor KeyInjector: InjectorProtocol { waitForTargetActivation: @escaping @Sendable (NSRunningApplication) async -> Bool = { _ in true }, isAccessibilityTrusted: @escaping @Sendable () -> Bool = { true }, hasEditableTarget: @escaping @Sendable () -> Bool = { true }, - isAXOpaqueEditor: @escaping @Sendable (NSRunningApplication?) -> Bool = { _ in false }, + isAXOpaqueApp: @escaping @Sendable (NSRunningApplication?) -> Bool = { _ in false }, clipboard: any ClipboardAccess = SystemClipboard() ) { self.pasteSettleDuration = pasteSettleDuration @@ -132,7 +132,7 @@ public actor KeyInjector: InjectorProtocol { self.waitForTargetActivation = waitForTargetActivation self.isAccessibilityTrusted = isAccessibilityTrusted self.hasEditableTarget = hasEditableTarget - self.isAXOpaqueEditor = isAXOpaqueEditor + self.isAXOpaqueApp = isAXOpaqueApp self.clipboard = clipboard } @@ -237,7 +237,7 @@ public actor KeyInjector: InjectorProtocol { // app — an Electron editor (VS Code, Slack) or a web browser — which can // report no editable signal even for a real text field; there we still paste // rather than drop the user's words. - guard hasEditableTarget() || isAXOpaqueEditor(target) else { + guard hasEditableTarget() || isAXOpaqueApp(target) else { clipboard.write(finalText) throw BlurtError.noEditableTarget } diff --git a/Tests/BlurtEngineTests/KeyInjectorInsertTests.swift b/Tests/BlurtEngineTests/KeyInjectorInsertTests.swift index 2c41b16..4dcbbe8 100644 --- a/Tests/BlurtEngineTests/KeyInjectorInsertTests.swift +++ b/Tests/BlurtEngineTests/KeyInjectorInsertTests.swift @@ -164,7 +164,7 @@ struct KeyInjectorInsertTests { return true }, hasEditableTarget: { false }, // Electron/Chromium exposes no editable AX signal - isAXOpaqueEditor: { _ in true }, // …but it *is* an Electron editor + isAXOpaqueApp: { _ in true }, // …but it *is* an AX-opaque app (Electron editor) clipboard: clip) // Must not throw noEditableTarget: the Electron exception keeps the paste.