From ac9b41574e877bb4604db9037cf82c843fc6637f Mon Sep 17 00:00:00 2001 From: "synacktra.work@gmail.com" Date: Thu, 9 Jul 2026 15:01:50 +0530 Subject: [PATCH 1/4] fix(handoff): re-probe completion state on operator connect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Detection listeners (framenavigated, DOM mutations) only fire on new events, so any transition into the completion state between session registration and first-connect was lost — the handoff would then wait for a future event that never came and time out on completion_timeout. Re-check on connect (LLM included now that the wrapper is loaded) and short-circuit if the state is already reached. Co-Authored-By: Claude Opus 4.7 --- browser_handoff/handoff.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/browser_handoff/handoff.py b/browser_handoff/handoff.py index 208c2c4..6e27ee7 100644 --- a/browser_handoff/handoff.py +++ b/browser_handoff/handoff.py @@ -601,9 +601,21 @@ async def on_completion_detected(detection: BaseDetection) -> None: timeout_cause: Literal["access", "completion"] | None = None async def install_listeners_after_connect() -> None: + nonlocal completion_reason await session.presence.wait_until_connected() if completion_event.is_set(): return + # Race defense: state may have been reached between the + # initial probe (T0) and first-connect. Listeners only + # fire on new events, so any transition that already + # happened would be missed. Re-probe here — now with LLM + # included, since the wrapper has loaded and vision + # calls are no longer wasted. + arrival = await until.check(page, reason=session.reason) + if arrival.matched: + completion_reason = arrival.reason + completion_event.set() + return listener_cleanups.append( until.register_listeners(page, on_completion_detected) ) From 7a9b723a186f1afa1f22ef6b68e77020f529eef3 Mon Sep 17 00:00:00 2001 From: "synacktra.work@gmail.com" Date: Thu, 9 Jul 2026 15:40:50 +0530 Subject: [PATCH 2/4] feat(server): fade viewer out under the completion card Previously the substrate iframe (passthrough) kept live focus and accepted keystrokes after completion, and the last stream frame lingered in memory (streaming). Now: defocus the iframe immediately so pending keystrokes stop hitting the substrate, keep the underlying frame visible for the ~300ms fade-in so the page fades under the card instead of snapping to black, then blank the iframe / drop the last frame. Applied symmetrically to both templates. Co-Authored-By: Claude Opus 4.7 --- browser_handoff/templates/intervention.html | 13 +++++++++++++ browser_handoff/templates/proxy_intervention.html | 12 ++++++++++++ 2 files changed, 25 insertions(+) diff --git a/browser_handoff/templates/intervention.html b/browser_handoff/templates/intervention.html index 2a267fe..59bdef8 100644 --- a/browser_handoff/templates/intervention.html +++ b/browser_handoff/templates/intervention.html @@ -1350,6 +1350,19 @@

Keyboard shortcuts

completionReasonText.textContent = reason || ''; completionTimestamp.textContent = nowTimeText(); completionOverlay.classList.add('show'); + // Keep the last frame visible for the fade-in so the page fades + // under the overlay rather than snapping to black, then drop it + // — mirrors the iframe-blank behavior in passthrough mode. + // Removing has-frame flips visibility hidden (no broken-image + // flash) and revoking the object URL releases the last blob. + setTimeout(() => { + stream.classList.remove('has-frame'); + stream.removeAttribute('src'); + if (currentObjectUrl) { + URL.revokeObjectURL(currentObjectUrl); + currentObjectUrl = null; + } + }, 350); markEnded(); } diff --git a/browser_handoff/templates/proxy_intervention.html b/browser_handoff/templates/proxy_intervention.html index 5ae2716..293873e 100644 --- a/browser_handoff/templates/proxy_intervention.html +++ b/browser_handoff/templates/proxy_intervention.html @@ -985,6 +985,18 @@

Keyboard shortcuts

completionReasonText.textContent = reason || ''; completionTimestamp.textContent = nowTimeText(); completionOverlay.classList.add('show'); + // Defocus the iframe immediately so any pending keystrokes stop + // hitting the substrate the moment the card appears. Keep the + // iframe visible for the fade-in so the page fades under the + // overlay rather than snapping to black, then blank it so the + // substrate bh no longer owns can't be interacted with even if + // the operator programmatically refocuses. + if (document.activeElement && document.activeElement.blur) { + document.activeElement.blur(); + } + setTimeout(() => { + try { iframe.src = 'about:blank'; } catch (_) {} + }, 350); markTerminal(); } From d0bcd5acf8783344f3a08965ac821a942f3e2741 Mon Sep 17 00:00:00 2001 From: "synacktra.work@gmail.com" Date: Thu, 9 Jul 2026 15:46:59 +0530 Subject: [PATCH 3/4] refactor(server): rename templates by mode, move under server/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - proxy_intervention.html -> server/templates/passthrough-mode.html - intervention.html -> server/templates/screencast-mode.html The old names ("intervention" as a generic verb; "proxy" f actually an iframe of a substrate viewer) obscured what each template is for. Naming them by mode makes the two-mode design visi tree. Living under server/ also disambiguates them from the top-level package — they're a server-owned rendering detail. Terminology harmonized across handoff.py, session.py, stre the templates' internal cross-references, tests, README, and example smoke tests. StreamingServer keeps its name — it hosts bot Co-Authored-By: Claude Opus 4.7 --- README.md | 2 +- browser_handoff/handoff.py | 8 +- browser_handoff/server/session.py | 4 +- browser_handoff/server/streaming.py | 22 +- .../server/templates/passthrough-mode.html | 1167 ++++++++++++ .../server/templates/screencast-mode.html | 1664 +++++++++++++++++ tests/integration/test_passthrough.py | 16 +- tests/test_server.py | 12 +- 8 files changed, 2863 insertions(+), 32 deletions(-) create mode 100644 browser_handoff/server/templates/passthrough-mode.html create mode 100644 browser_handoff/server/templates/screencast-mode.html diff --git a/README.md b/README.md index 17b341a..974d60e 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ On timeout the result has `timed_out=True` and `timeout_cause` names the timer t ## Passthrough mode (cloud substrates) -When the page lives in a cloud browser substrate (Kernel, Browserbase, Steel, Cua, …) and `browser-handoff` runs on your machine, streaming mode relays every frame and every operator input through your local process — each interaction round-trips over the WAN between your machine and the substrate. Frames tolerate it; input doesn't. Observed input latency against a Kernel cloud browser in streaming mode was ~30 seconds per keystroke — unusable for filling out a form. +When the page lives in a cloud browser substrate (Kernel, Browserbase, Steel, Cua, …) and `browser-handoff` runs on your machine, screencast mode relays every frame and every operator input through your local process — each interaction round-trips over the WAN between your machine and the substrate. Frames tolerate it; input doesn't. Observed input latency against a Kernel cloud browser in screencast mode was ~30 seconds per keystroke — unusable for filling out a form. Most substrates already ship their own first-class viewer. Passthrough mode delegates streaming and input to that viewer while `browser-handoff` keeps the detection, notification, and lifecycle responsibilities. diff --git a/browser_handoff/handoff.py b/browser_handoff/handoff.py index 6e27ee7..a6cbdd5 100644 --- a/browser_handoff/handoff.py +++ b/browser_handoff/handoff.py @@ -32,8 +32,8 @@ DEFAULT_VIEWPORT = {"width": 1280, "height": 800} -# Read the page's rect on the substrate's display so the proxy template -# can crop the iframe to just the page area (the substrate streams the +# Read the page's rect on the substrate's display so the passthrough +# template can crop the iframe to just the page area (the substrate streams the # whole desktop). page_y accounts for browser chrome via # `outerHeight - innerHeight`; page_x mirrors that in case of symmetric # window borders. @@ -97,7 +97,7 @@ async def _capture_crop_metrics( Returns None when the evaluate raises, the page reports zero dims even after retries, or the substrate mocks screen dims (headless). - On None, the proxy template falls back to a non-cropped iframe. + On None, the passthrough template falls back to a non-cropped iframe. """ await _maximize_substrate_window(page) @@ -553,7 +553,7 @@ async def on_completion_detected(detection: BaseDetection) -> None: except Exception as e: logger.info(f"Could not get viewport: {e}, using default: {viewport_size}") - # Page-rect-on-display metrics for the proxy template's + # Page-rect-on-display metrics for the passthrough template's # iframe crop. Only used in passthrough mode. crop_metrics: dict[str, int] | None = None if stream_url is not None: diff --git a/browser_handoff/server/session.py b/browser_handoff/server/session.py index 830feb6..059b804 100644 --- a/browser_handoff/server/session.py +++ b/browser_handoff/server/session.py @@ -110,8 +110,8 @@ class HandoffSession: # bh still owns detection + notification + lifecycle. stream_url: str | None = None # Page rect on the substrate's display (six ints: screen_w/h, - # page_x/y, page_w/h). Used by the proxy template's CSS to crop the - # iframe to just the page content. None when not in passthrough mode + # page_x/y, page_w/h). Used by the passthrough template's CSS to crop + # the iframe to just the page content. None when not in passthrough mode # or when the JS evaluate returned degenerate values. crop_metrics: dict[str, int] | None = None # Per-session resolved timeouts. None at either layer means "no diff --git a/browser_handoff/server/streaming.py b/browser_handoff/server/streaming.py index e86463c..19bd77c 100644 --- a/browser_handoff/server/streaming.py +++ b/browser_handoff/server/streaming.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: from playwright.async_api import BrowserContext, CDPSession, Page -TEMPLATE_DIR = Path(__file__).parent.parent / "templates" +TEMPLATE_DIR = Path(__file__).parent / "templates" jinja_env = Environment(loader=FileSystemLoader(TEMPLATE_DIR), autoescape=True) # Suppress the native right-click menu — it's an OS-level overlay the @@ -691,7 +691,7 @@ async def _handle_passthrough_websocket( elif msg_type == "presence": # ~2s heartbeat while the tab is visible. session.presence.bump() - # Other types are silently ignored — streaming-mode + # Other types are silently ignored — screencast-mode # messages (mouse/keyboard/…) don't apply here. except Exception as e: logger.error(f"passthrough WebSocket error: {e}") @@ -1085,7 +1085,7 @@ async def _capture_session_end_screenshot( ) -> str | None: """Snapshot the page as a base64 JPEG data URL for session-end events. - Only meaningful in passthrough mode — streaming mode keeps its + Only meaningful in passthrough mode — screencast mode keeps its last screencast frame on display. Returns None on any failure. """ if not session.is_passthrough: @@ -1119,9 +1119,9 @@ async def _broadcast_session_end( async def notify_task_expired(self, session_id: str) -> None: """Push a task_expired event — the human didn't finish in time. - Distinct from cancellation: this is the timeout path. The proxy - template swaps the iframe out for the captured screenshot; - streaming-mode UI ignores this event. + Distinct from cancellation: this is the timeout path. The + passthrough template swaps the iframe out for the captured + screenshot; screencast-mode UI ignores this event. """ await self._broadcast_session_end(session_id, "task_expired") @@ -1148,13 +1148,13 @@ async def stop_screencast(self, session_id: str) -> None: def _get_html_client(self, session_id: str, reason: str) -> str: """Render the operator HTML for a session. - Passthrough sessions get `proxy_intervention.html` (iframes the - substrate viewer + crops via crop_metrics); streaming sessions - get `intervention.html` (the screencast viewer). + Passthrough sessions get `passthrough-mode.html` (iframes the + substrate viewer + crops via crop_metrics); screencast sessions + get `screencast-mode.html` (the CDP-screencast viewer). """ session = self.sessions[session_id] if session.is_passthrough: - template = jinja_env.get_template("proxy_intervention.html") + template = jinja_env.get_template("passthrough-mode.html") return template.render( access_token=session.access_token, reason=reason, @@ -1164,7 +1164,7 @@ def _get_html_client(self, session_id: str, reason: str) -> str: stream_url=session.stream_url, crop_metrics=session.crop_metrics, ) - template = jinja_env.get_template("intervention.html") + template = jinja_env.get_template("screencast-mode.html") return template.render( access_token=session.access_token, reason=reason, diff --git a/browser_handoff/server/templates/passthrough-mode.html b/browser_handoff/server/templates/passthrough-mode.html new file mode 100644 index 0000000..efeebe3 --- /dev/null +++ b/browser_handoff/server/templates/passthrough-mode.html @@ -0,0 +1,1167 @@ + + + + + + Browser Handoff — {{ scenario_name or 'Human Intervention' }} + + + + + + +
+
+ + + browser-handoff + + {% if scenario_name %} + / + {{ scenario_name }} + {% endif %} + / + + {{ reason }} + +
+ +
+ +
+
+ + + + +
+ + + + + + + + + + + +
+ +
+ --:-- +
+ +
+ + Connecting +
+
+ +
+
Loading remote stream…
+ + +
+
+

Keyboard shortcuts

+
+ Show / hide this overlay + ? +
+
+ Close overlay + Esc +
+
+ Reload the page + Click reload icon +
+
+ Mouse, keyboard, copy/paste, and other in-page interactions are + handled by the embedded viewer below. Click into the viewer + first, then interact normally. +
+
+
+ +
+
+
+ + + + + + Completed + +
+
+
You may close this window
+
+
+ +
+
+
+ + + + + + + Session expired + +
+
+ Your time to complete the handoff ran out. +
+
You may close this window
+
+
+ +
+
+
+ + + + + + + Session ended + +
+
+ The handoff session ended. +
+
You may close this window
+
+
+ + Last page state before session ended +
+
+
+ + + + diff --git a/browser_handoff/server/templates/screencast-mode.html b/browser_handoff/server/templates/screencast-mode.html new file mode 100644 index 0000000..6718e34 --- /dev/null +++ b/browser_handoff/server/templates/screencast-mode.html @@ -0,0 +1,1664 @@ + + + + + + Browser Handoff — {{ scenario_name or 'Human Intervention' }} + + + + + + +
+
+ + + browser-handoff + + {% if scenario_name %} + / + {{ scenario_name }} + {% endif %} + / + + {{ reason }} + +
+ +
+ +
+
+ + + + +
+ + + + + + + + + + + +
+ +
+ --:-- +
+ +
+ + Connecting +
+
+ +
+ Browser Stream + +
+
Connecting
+
+ +
+ +
+
+

Keyboard shortcuts

+
+ Paste from clipboard + CtrlV +
+
+ Copy selection + CtrlC +
+
+ Cut selection + CtrlX +
+
+ Select all + CtrlA +
+
+ Reload page + Click reload icon +
+
+ Close this overlay + Esc or ? +
+
+
+ +
+
+
+ + + + + + Completed + +
+
+
You may close this window
+
+
+ +
+
+
+ + + + + + + Session expired + +
+
+ Your time to complete the handoff ran out. +
+
You may close this window
+
+
+ +
+
+
+ + + + + + + Session ended + +
+
+ The handoff session ended. +
+
You may close this window
+
+
+
+
+
+ + + + diff --git a/tests/integration/test_passthrough.py b/tests/integration/test_passthrough.py index 96c110a..dc2151c 100644 --- a/tests/integration/test_passthrough.py +++ b/tests/integration/test_passthrough.py @@ -4,9 +4,9 @@ plumbing end-to-end: - The CDP screencast task is NOT started when stream_url is set. - - GET /?t= serves the proxy template (not the streaming one). + - GET /?t= serves the passthrough template (not the screencast one). - The status WebSocket dispatches to the passthrough handler. - - notify_task_expired delivers an event the proxy template can react to. + - notify_task_expired delivers an event the passthrough template can react to. In-page activity observation moved into LLMDetection's unified watcher after the v0.6 refactor — the stealth observer and bump behavior are @@ -51,7 +51,7 @@ async def test_passthrough_skips_screencast_pump( """When stream_url is set, register_session must not schedule capture_task. The session's frame_seq stays at 0 (no frames produced) and capture_task - remains None. Streaming-mode comparison test in test_screencast_input.py + remains None. Screencast-mode comparison test in test_screencast_input.py confirms the inverse — pump runs without stream_url. """ port = _free_port() @@ -96,10 +96,10 @@ async def test_passthrough_skips_screencast_pump( await ctx.close() -async def test_passthrough_serves_proxy_template( +async def test_passthrough_serves_passthrough_template( browser: Browser, base_url: str ) -> None: - """GET /?t= returns the proxy template, not intervention.html.""" + """GET /?t= returns passthrough-mode.html, not screencast-mode.html.""" port = _free_port() h = Handoff(server=ServerConfig(host="127.0.0.1", port=port)) @@ -111,7 +111,7 @@ async def test_passthrough_serves_proxy_template( h.pause( page, until=Detection.url(path_contains=["/dashboard"]), - reason="proxy template test", + reason="passthrough template test", stream_url="https://dummy.substrate.example/viewer?t=xyz", ) ) @@ -124,10 +124,10 @@ async def test_passthrough_serves_proxy_template( # route uses (templates are static once the session is registered). # Avoids a second HTTP-client dep just to verify the response body. html = server._get_html_client(session.session_id, session.reason) - # Proxy-only markers; would not appear in intervention.html. + # Passthrough-only markers; would not appear in screencast-mode.html. assert "substrate-iframe" in html assert "fallback-screenshot" in html - assert "proxy template test" in html + assert "passthrough template test" in html finally: # Simulate operator opening the wrapper. One bump flips the # connect gate AND records the freshness timestamp. diff --git a/tests/test_server.py b/tests/test_server.py index 5c18d12..bfe23bd 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -333,17 +333,17 @@ def _register(self, server: StreamingServer, **overrides): server._token_to_session[session.access_token] = session.session_id return session - def test_streaming_session_renders_intervention_template(self): + def test_screencast_session_renders_screencast_mode_template(self): server = StreamingServer() self._register(server) html = server._get_html_client("test", "please log in") - # intervention.html ships the streaming-mode features that the - # proxy template intentionally omits. + # screencast-mode.html ships the screencast-mode features that + # the passthrough template intentionally omits. assert "Browser Handoff" in html assert "please log in" in html assert "stream-container" in html # streaming-only element id - def test_passthrough_session_renders_proxy_template(self): + def test_passthrough_session_renders_passthrough_mode_template(self): server = StreamingServer() crop = { "screen_w": 1920, "screen_h": 1080, @@ -357,10 +357,10 @@ def test_passthrough_session_renders_proxy_template(self): ) html = server._get_html_client("test", "please sign in") assert "please sign in" in html - # Proxy-only markers: the substrate iframe and the fallback + # Passthrough-only markers: the substrate iframe and the fallback # screenshot used when the bh session ends without completion # (substrate's WebRTC stream would otherwise keep running in the - # iframe). Neither exists in intervention.html. + # iframe). Neither exists in screencast-mode.html. assert "substrate-iframe" in html assert "fallback-screenshot" in html # Crop metrics threaded into the CSS via Jinja. From 7df5618b6afb4263d56f870d8b42e9ecdef79853 Mon Sep 17 00:00:00 2001 From: "synacktra.work@gmail.com" Date: Thu, 9 Jul 2026 15:48:10 +0530 Subject: [PATCH 4/4] chore(server): drop deprecated get_stream_url shim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Its docstring said "Removed in v0.7" while the warning said "Will be removed in v0.7" — self-contradictory in v0.7.x. Resolving toward the docstring: it's gone. Callers were migrated to get_operator_url before v0.7, so no live consumers remain. Co-Authored-By: Claude Opus 4.7 --- browser_handoff/server/streaming.py | 12 ------------ tests/test_server.py | 13 ------------- 2 files changed, 25 deletions(-) diff --git a/browser_handoff/server/streaming.py b/browser_handoff/server/streaming.py index 19bd77c..d3aa538 100644 --- a/browser_handoff/server/streaming.py +++ b/browser_handoff/server/streaming.py @@ -1187,18 +1187,6 @@ def get_operator_url(self, session_id: str) -> str: token = self.sessions[session_id].access_token return f"{base_url}/?t={token}" - def get_stream_url(self, session_id: str) -> str: - """Deprecated alias for :meth:`get_operator_url`. Removed in v0.7.""" - import warnings - - warnings.warn( - "get_stream_url() is deprecated; use get_operator_url() instead. " - "Will be removed in v0.7.", - DeprecationWarning, - stacklevel=2, - ) - return self.get_operator_url(session_id) - async def start(self) -> None: """Bind the port and serve until `stop()` is called.""" config = uvicorn.Config( diff --git a/tests/test_server.py b/tests/test_server.py index bfe23bd..0ba9aa6 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -257,19 +257,6 @@ def test_operator_url_carries_token_not_session_id(self): assert f"?t={session.access_token}" in url assert "?session=" not in url # the id is no longer the URL gate - def test_get_stream_url_is_deprecated_alias(self): - import warnings - - server = StreamingServer() - session = self._register(server, expires_at=time.time() + 60) - canonical = server.get_operator_url(session.session_id) - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - legacy = server.get_stream_url(session.session_id) - assert legacy == canonical - assert any(issubclass(w.category, DeprecationWarning) for w in caught) - - class TestPassthroughSession: """HandoffSession.is_passthrough is derived from stream_url.