diff --git a/README.md b/README.md index 17b341a..974d60e 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ On timeout the result has `timed_out=True` and `timeout_cause` names the timer t ## Passthrough mode (cloud substrates) -When the page lives in a cloud browser substrate (Kernel, Browserbase, Steel, Cua, …) and `browser-handoff` runs on your machine, streaming mode relays every frame and every operator input through your local process — each interaction round-trips over the WAN between your machine and the substrate. Frames tolerate it; input doesn't. Observed input latency against a Kernel cloud browser in streaming mode was ~30 seconds per keystroke — unusable for filling out a form. +When the page lives in a cloud browser substrate (Kernel, Browserbase, Steel, Cua, …) and `browser-handoff` runs on your machine, screencast mode relays every frame and every operator input through your local process — each interaction round-trips over the WAN between your machine and the substrate. Frames tolerate it; input doesn't. Observed input latency against a Kernel cloud browser in screencast mode was ~30 seconds per keystroke — unusable for filling out a form. Most substrates already ship their own first-class viewer. Passthrough mode delegates streaming and input to that viewer while `browser-handoff` keeps the detection, notification, and lifecycle responsibilities. diff --git a/browser_handoff/handoff.py b/browser_handoff/handoff.py index 208c2c4..a6cbdd5 100644 --- a/browser_handoff/handoff.py +++ b/browser_handoff/handoff.py @@ -32,8 +32,8 @@ DEFAULT_VIEWPORT = {"width": 1280, "height": 800} -# Read the page's rect on the substrate's display so the proxy template -# can crop the iframe to just the page area (the substrate streams the +# Read the page's rect on the substrate's display so the passthrough +# template can crop the iframe to just the page area (the substrate streams the # whole desktop). page_y accounts for browser chrome via # `outerHeight - innerHeight`; page_x mirrors that in case of symmetric # window borders. @@ -97,7 +97,7 @@ async def _capture_crop_metrics( Returns None when the evaluate raises, the page reports zero dims even after retries, or the substrate mocks screen dims (headless). - On None, the proxy template falls back to a non-cropped iframe. + On None, the passthrough template falls back to a non-cropped iframe. """ await _maximize_substrate_window(page) @@ -553,7 +553,7 @@ async def on_completion_detected(detection: BaseDetection) -> None: except Exception as e: logger.info(f"Could not get viewport: {e}, using default: {viewport_size}") - # Page-rect-on-display metrics for the proxy template's + # Page-rect-on-display metrics for the passthrough template's # iframe crop. Only used in passthrough mode. crop_metrics: dict[str, int] | None = None if stream_url is not None: @@ -601,9 +601,21 @@ async def on_completion_detected(detection: BaseDetection) -> None: timeout_cause: Literal["access", "completion"] | None = None async def install_listeners_after_connect() -> None: + nonlocal completion_reason await session.presence.wait_until_connected() if completion_event.is_set(): return + # Race defense: state may have been reached between the + # initial probe (T0) and first-connect. Listeners only + # fire on new events, so any transition that already + # happened would be missed. Re-probe here — now with LLM + # included, since the wrapper has loaded and vision + # calls are no longer wasted. + arrival = await until.check(page, reason=session.reason) + if arrival.matched: + completion_reason = arrival.reason + completion_event.set() + return listener_cleanups.append( until.register_listeners(page, on_completion_detected) ) diff --git a/browser_handoff/server/session.py b/browser_handoff/server/session.py index 830feb6..059b804 100644 --- a/browser_handoff/server/session.py +++ b/browser_handoff/server/session.py @@ -110,8 +110,8 @@ class HandoffSession: # bh still owns detection + notification + lifecycle. stream_url: str | None = None # Page rect on the substrate's display (six ints: screen_w/h, - # page_x/y, page_w/h). Used by the proxy template's CSS to crop the - # iframe to just the page content. None when not in passthrough mode + # page_x/y, page_w/h). Used by the passthrough template's CSS to crop + # the iframe to just the page content. None when not in passthrough mode # or when the JS evaluate returned degenerate values. crop_metrics: dict[str, int] | None = None # Per-session resolved timeouts. None at either layer means "no diff --git a/browser_handoff/server/streaming.py b/browser_handoff/server/streaming.py index e86463c..d3aa538 100644 --- a/browser_handoff/server/streaming.py +++ b/browser_handoff/server/streaming.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: from playwright.async_api import BrowserContext, CDPSession, Page -TEMPLATE_DIR = Path(__file__).parent.parent / "templates" +TEMPLATE_DIR = Path(__file__).parent / "templates" jinja_env = Environment(loader=FileSystemLoader(TEMPLATE_DIR), autoescape=True) # Suppress the native right-click menu — it's an OS-level overlay the @@ -691,7 +691,7 @@ async def _handle_passthrough_websocket( elif msg_type == "presence": # ~2s heartbeat while the tab is visible. session.presence.bump() - # Other types are silently ignored — streaming-mode + # Other types are silently ignored — screencast-mode # messages (mouse/keyboard/…) don't apply here. except Exception as e: logger.error(f"passthrough WebSocket error: {e}") @@ -1085,7 +1085,7 @@ async def _capture_session_end_screenshot( ) -> str | None: """Snapshot the page as a base64 JPEG data URL for session-end events. - Only meaningful in passthrough mode — streaming mode keeps its + Only meaningful in passthrough mode — screencast mode keeps its last screencast frame on display. Returns None on any failure. """ if not session.is_passthrough: @@ -1119,9 +1119,9 @@ async def _broadcast_session_end( async def notify_task_expired(self, session_id: str) -> None: """Push a task_expired event — the human didn't finish in time. - Distinct from cancellation: this is the timeout path. The proxy - template swaps the iframe out for the captured screenshot; - streaming-mode UI ignores this event. + Distinct from cancellation: this is the timeout path. The + passthrough template swaps the iframe out for the captured + screenshot; screencast-mode UI ignores this event. """ await self._broadcast_session_end(session_id, "task_expired") @@ -1148,13 +1148,13 @@ async def stop_screencast(self, session_id: str) -> None: def _get_html_client(self, session_id: str, reason: str) -> str: """Render the operator HTML for a session. - Passthrough sessions get `proxy_intervention.html` (iframes the - substrate viewer + crops via crop_metrics); streaming sessions - get `intervention.html` (the screencast viewer). + Passthrough sessions get `passthrough-mode.html` (iframes the + substrate viewer + crops via crop_metrics); screencast sessions + get `screencast-mode.html` (the CDP-screencast viewer). """ session = self.sessions[session_id] if session.is_passthrough: - template = jinja_env.get_template("proxy_intervention.html") + template = jinja_env.get_template("passthrough-mode.html") return template.render( access_token=session.access_token, reason=reason, @@ -1164,7 +1164,7 @@ def _get_html_client(self, session_id: str, reason: str) -> str: stream_url=session.stream_url, crop_metrics=session.crop_metrics, ) - template = jinja_env.get_template("intervention.html") + template = jinja_env.get_template("screencast-mode.html") return template.render( access_token=session.access_token, reason=reason, @@ -1187,18 +1187,6 @@ def get_operator_url(self, session_id: str) -> str: token = self.sessions[session_id].access_token return f"{base_url}/?t={token}" - def get_stream_url(self, session_id: str) -> str: - """Deprecated alias for :meth:`get_operator_url`. Removed in v0.7.""" - import warnings - - warnings.warn( - "get_stream_url() is deprecated; use get_operator_url() instead. " - "Will be removed in v0.7.", - DeprecationWarning, - stacklevel=2, - ) - return self.get_operator_url(session_id) - async def start(self) -> None: """Bind the port and serve until `stop()` is called.""" config = uvicorn.Config( diff --git a/browser_handoff/server/templates/passthrough-mode.html b/browser_handoff/server/templates/passthrough-mode.html new file mode 100644 index 0000000..efeebe3 --- /dev/null +++ b/browser_handoff/server/templates/passthrough-mode.html @@ -0,0 +1,1167 @@ + + + + + + Browser Handoff — {{ scenario_name or 'Human Intervention' }} + + + + + + +
+
+ + + browser-handoff + + {% if scenario_name %} + / + {{ scenario_name }} + {% endif %} + / + + {{ reason }} + +
+ +
+ +
+
+ + + + +
+ + + + + + + + + + + +
+ +
+ --:-- +
+ +
+ + Connecting +
+
+ +
+
Loading remote stream…
+ + +
+
+

Keyboard shortcuts

+
+ Show / hide this overlay + ? +
+
+ Close overlay + Esc +
+
+ Reload the page + Click reload icon +
+
+ Mouse, keyboard, copy/paste, and other in-page interactions are + handled by the embedded viewer below. Click into the viewer + first, then interact normally. +
+
+
+ +
+
+
+ + + + + + Completed + +
+
+
You may close this window
+
+
+ +
+
+
+ + + + + + + Session expired + +
+
+ Your time to complete the handoff ran out. +
+
You may close this window
+
+
+ +
+
+
+ + + + + + + Session ended + +
+
+ The handoff session ended. +
+
You may close this window
+
+
+ + Last page state before session ended +
+
+
+ + + + diff --git a/browser_handoff/server/templates/screencast-mode.html b/browser_handoff/server/templates/screencast-mode.html new file mode 100644 index 0000000..6718e34 --- /dev/null +++ b/browser_handoff/server/templates/screencast-mode.html @@ -0,0 +1,1664 @@ + + + + + + Browser Handoff — {{ scenario_name or 'Human Intervention' }} + + + + + + +
+
+ + + browser-handoff + + {% if scenario_name %} + / + {{ scenario_name }} + {% endif %} + / + + {{ reason }} + +
+ +
+ +
+
+ + + + +
+ + + + + + + + + + + +
+ +
+ --:-- +
+ +
+ + Connecting +
+
+ +
+ Browser Stream + +
+
Connecting
+
+ +
+ +
+
+

Keyboard shortcuts

+
+ Paste from clipboard + CtrlV +
+
+ Copy selection + CtrlC +
+
+ Cut selection + CtrlX +
+
+ Select all + CtrlA +
+
+ Reload page + Click reload icon +
+
+ Close this overlay + Esc or ? +
+
+
+ +
+
+
+ + + + + + Completed + +
+
+
You may close this window
+
+
+ +
+
+
+ + + + + + + Session expired + +
+
+ Your time to complete the handoff ran out. +
+
You may close this window
+
+
+ +
+
+
+ + + + + + + Session ended + +
+
+ The handoff session ended. +
+
You may close this window
+
+
+
+
+
+ + + + diff --git a/browser_handoff/templates/intervention.html b/browser_handoff/templates/intervention.html index 2a267fe..59bdef8 100644 --- a/browser_handoff/templates/intervention.html +++ b/browser_handoff/templates/intervention.html @@ -1350,6 +1350,19 @@

Keyboard shortcuts

completionReasonText.textContent = reason || ''; completionTimestamp.textContent = nowTimeText(); completionOverlay.classList.add('show'); + // Keep the last frame visible for the fade-in so the page fades + // under the overlay rather than snapping to black, then drop it + // — mirrors the iframe-blank behavior in passthrough mode. + // Removing has-frame flips visibility hidden (no broken-image + // flash) and revoking the object URL releases the last blob. + setTimeout(() => { + stream.classList.remove('has-frame'); + stream.removeAttribute('src'); + if (currentObjectUrl) { + URL.revokeObjectURL(currentObjectUrl); + currentObjectUrl = null; + } + }, 350); markEnded(); } diff --git a/browser_handoff/templates/proxy_intervention.html b/browser_handoff/templates/proxy_intervention.html index 5ae2716..293873e 100644 --- a/browser_handoff/templates/proxy_intervention.html +++ b/browser_handoff/templates/proxy_intervention.html @@ -985,6 +985,18 @@

Keyboard shortcuts

completionReasonText.textContent = reason || ''; completionTimestamp.textContent = nowTimeText(); completionOverlay.classList.add('show'); + // Defocus the iframe immediately so any pending keystrokes stop + // hitting the substrate the moment the card appears. Keep the + // iframe visible for the fade-in so the page fades under the + // overlay rather than snapping to black, then blank it so the + // substrate bh no longer owns can't be interacted with even if + // the operator programmatically refocuses. + if (document.activeElement && document.activeElement.blur) { + document.activeElement.blur(); + } + setTimeout(() => { + try { iframe.src = 'about:blank'; } catch (_) {} + }, 350); markTerminal(); } diff --git a/tests/integration/test_passthrough.py b/tests/integration/test_passthrough.py index 96c110a..dc2151c 100644 --- a/tests/integration/test_passthrough.py +++ b/tests/integration/test_passthrough.py @@ -4,9 +4,9 @@ plumbing end-to-end: - The CDP screencast task is NOT started when stream_url is set. - - GET /?t= serves the proxy template (not the streaming one). + - GET /?t= serves the passthrough template (not the screencast one). - The status WebSocket dispatches to the passthrough handler. - - notify_task_expired delivers an event the proxy template can react to. + - notify_task_expired delivers an event the passthrough template can react to. In-page activity observation moved into LLMDetection's unified watcher after the v0.6 refactor — the stealth observer and bump behavior are @@ -51,7 +51,7 @@ async def test_passthrough_skips_screencast_pump( """When stream_url is set, register_session must not schedule capture_task. The session's frame_seq stays at 0 (no frames produced) and capture_task - remains None. Streaming-mode comparison test in test_screencast_input.py + remains None. Screencast-mode comparison test in test_screencast_input.py confirms the inverse — pump runs without stream_url. """ port = _free_port() @@ -96,10 +96,10 @@ async def test_passthrough_skips_screencast_pump( await ctx.close() -async def test_passthrough_serves_proxy_template( +async def test_passthrough_serves_passthrough_template( browser: Browser, base_url: str ) -> None: - """GET /?t= returns the proxy template, not intervention.html.""" + """GET /?t= returns passthrough-mode.html, not screencast-mode.html.""" port = _free_port() h = Handoff(server=ServerConfig(host="127.0.0.1", port=port)) @@ -111,7 +111,7 @@ async def test_passthrough_serves_proxy_template( h.pause( page, until=Detection.url(path_contains=["/dashboard"]), - reason="proxy template test", + reason="passthrough template test", stream_url="https://dummy.substrate.example/viewer?t=xyz", ) ) @@ -124,10 +124,10 @@ async def test_passthrough_serves_proxy_template( # route uses (templates are static once the session is registered). # Avoids a second HTTP-client dep just to verify the response body. html = server._get_html_client(session.session_id, session.reason) - # Proxy-only markers; would not appear in intervention.html. + # Passthrough-only markers; would not appear in screencast-mode.html. assert "substrate-iframe" in html assert "fallback-screenshot" in html - assert "proxy template test" in html + assert "passthrough template test" in html finally: # Simulate operator opening the wrapper. One bump flips the # connect gate AND records the freshness timestamp. diff --git a/tests/test_server.py b/tests/test_server.py index 5c18d12..0ba9aa6 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -257,19 +257,6 @@ def test_operator_url_carries_token_not_session_id(self): assert f"?t={session.access_token}" in url assert "?session=" not in url # the id is no longer the URL gate - def test_get_stream_url_is_deprecated_alias(self): - import warnings - - server = StreamingServer() - session = self._register(server, expires_at=time.time() + 60) - canonical = server.get_operator_url(session.session_id) - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - legacy = server.get_stream_url(session.session_id) - assert legacy == canonical - assert any(issubclass(w.category, DeprecationWarning) for w in caught) - - class TestPassthroughSession: """HandoffSession.is_passthrough is derived from stream_url. @@ -333,17 +320,17 @@ def _register(self, server: StreamingServer, **overrides): server._token_to_session[session.access_token] = session.session_id return session - def test_streaming_session_renders_intervention_template(self): + def test_screencast_session_renders_screencast_mode_template(self): server = StreamingServer() self._register(server) html = server._get_html_client("test", "please log in") - # intervention.html ships the streaming-mode features that the - # proxy template intentionally omits. + # screencast-mode.html ships the screencast-mode features that + # the passthrough template intentionally omits. assert "Browser Handoff" in html assert "please log in" in html assert "stream-container" in html # streaming-only element id - def test_passthrough_session_renders_proxy_template(self): + def test_passthrough_session_renders_passthrough_mode_template(self): server = StreamingServer() crop = { "screen_w": 1920, "screen_h": 1080, @@ -357,10 +344,10 @@ def test_passthrough_session_renders_proxy_template(self): ) html = server._get_html_client("test", "please sign in") assert "please sign in" in html - # Proxy-only markers: the substrate iframe and the fallback + # Passthrough-only markers: the substrate iframe and the fallback # screenshot used when the bh session ends without completion # (substrate's WebRTC stream would otherwise keep running in the - # iframe). Neither exists in intervention.html. + # iframe). Neither exists in screencast-mode.html. assert "substrate-iframe" in html assert "fallback-screenshot" in html # Crop metrics threaded into the CSS via Jinja.