Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ On timeout the result has `timed_out=True` and `timeout_cause` names the timer t

## Passthrough mode (cloud substrates)

When the page lives in a cloud browser substrate (Kernel, Browserbase, Steel, Cua, …) and `browser-handoff` runs on your machine, streaming mode relays every frame and every operator input through your local process — each interaction round-trips over the WAN between your machine and the substrate. Frames tolerate it; input doesn't. Observed input latency against a Kernel cloud browser in streaming mode was ~30 seconds per keystroke — unusable for filling out a form.
When the page lives in a cloud browser substrate (Kernel, Browserbase, Steel, Cua, …) and `browser-handoff` runs on your machine, screencast mode relays every frame and every operator input through your local process — each interaction round-trips over the WAN between your machine and the substrate. Frames tolerate it; input doesn't. Observed input latency against a Kernel cloud browser in screencast mode was ~30 seconds per keystroke — unusable for filling out a form.

Most substrates already ship their own first-class viewer. Passthrough mode delegates streaming and input to that viewer while `browser-handoff` keeps the detection, notification, and lifecycle responsibilities.

Expand Down
20 changes: 16 additions & 4 deletions browser_handoff/handoff.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@

DEFAULT_VIEWPORT = {"width": 1280, "height": 800}

# Read the page's rect on the substrate's display so the proxy template
# can crop the iframe to just the page area (the substrate streams the
# Read the page's rect on the substrate's display so the passthrough
# template can crop the iframe to just the page area (the substrate streams the
# whole desktop). page_y accounts for browser chrome via
# `outerHeight - innerHeight`; page_x mirrors that in case of symmetric
# window borders.
Expand Down Expand Up @@ -97,7 +97,7 @@ async def _capture_crop_metrics(

Returns None when the evaluate raises, the page reports zero dims
even after retries, or the substrate mocks screen dims (headless).
On None, the proxy template falls back to a non-cropped iframe.
On None, the passthrough template falls back to a non-cropped iframe.
"""
await _maximize_substrate_window(page)

Expand Down Expand Up @@ -553,7 +553,7 @@ async def on_completion_detected(detection: BaseDetection) -> None:
except Exception as e:
logger.info(f"Could not get viewport: {e}, using default: {viewport_size}")

# Page-rect-on-display metrics for the proxy template's
# Page-rect-on-display metrics for the passthrough template's
# iframe crop. Only used in passthrough mode.
crop_metrics: dict[str, int] | None = None
if stream_url is not None:
Expand Down Expand Up @@ -601,9 +601,21 @@ async def on_completion_detected(detection: BaseDetection) -> None:
timeout_cause: Literal["access", "completion"] | None = None

async def install_listeners_after_connect() -> None:
nonlocal completion_reason
await session.presence.wait_until_connected()
if completion_event.is_set():
return
# Race defense: state may have been reached between the
# initial probe (T0) and first-connect. Listeners only
# fire on new events, so any transition that already
# happened would be missed. Re-probe here — now with LLM
# included, since the wrapper has loaded and vision
# calls are no longer wasted.
arrival = await until.check(page, reason=session.reason)
if arrival.matched:
completion_reason = arrival.reason
completion_event.set()
return
listener_cleanups.append(
until.register_listeners(page, on_completion_detected)
)
Expand Down
4 changes: 2 additions & 2 deletions browser_handoff/server/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,8 @@ class HandoffSession:
# bh still owns detection + notification + lifecycle.
stream_url: str | None = None
# Page rect on the substrate's display (six ints: screen_w/h,
# page_x/y, page_w/h). Used by the proxy template's CSS to crop the
# iframe to just the page content. None when not in passthrough mode
# page_x/y, page_w/h). Used by the passthrough template's CSS to crop
# the iframe to just the page content. None when not in passthrough mode
# or when the JS evaluate returned degenerate values.
crop_metrics: dict[str, int] | None = None
# Per-session resolved timeouts. None at either layer means "no
Expand Down
34 changes: 11 additions & 23 deletions browser_handoff/server/streaming.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
if TYPE_CHECKING:
from playwright.async_api import BrowserContext, CDPSession, Page

TEMPLATE_DIR = Path(__file__).parent.parent / "templates"
TEMPLATE_DIR = Path(__file__).parent / "templates"
jinja_env = Environment(loader=FileSystemLoader(TEMPLATE_DIR), autoescape=True)

# Suppress the native right-click menu — it's an OS-level overlay the
Expand Down Expand Up @@ -691,7 +691,7 @@ async def _handle_passthrough_websocket(
elif msg_type == "presence":
# ~2s heartbeat while the tab is visible.
session.presence.bump()
# Other types are silently ignored — streaming-mode
# Other types are silently ignored — screencast-mode
# messages (mouse/keyboard/…) don't apply here.
except Exception as e:
logger.error(f"passthrough WebSocket error: {e}")
Expand Down Expand Up @@ -1085,7 +1085,7 @@ async def _capture_session_end_screenshot(
) -> str | None:
"""Snapshot the page as a base64 JPEG data URL for session-end events.

Only meaningful in passthrough mode — streaming mode keeps its
Only meaningful in passthrough mode — screencast mode keeps its
last screencast frame on display. Returns None on any failure.
"""
if not session.is_passthrough:
Expand Down Expand Up @@ -1119,9 +1119,9 @@ async def _broadcast_session_end(
async def notify_task_expired(self, session_id: str) -> None:
"""Push a task_expired event — the human didn't finish in time.

Distinct from cancellation: this is the timeout path. The proxy
template swaps the iframe out for the captured screenshot;
streaming-mode UI ignores this event.
Distinct from cancellation: this is the timeout path. The
passthrough template swaps the iframe out for the captured
screenshot; screencast-mode UI ignores this event.
"""
await self._broadcast_session_end(session_id, "task_expired")

Expand All @@ -1148,13 +1148,13 @@ async def stop_screencast(self, session_id: str) -> None:
def _get_html_client(self, session_id: str, reason: str) -> str:
"""Render the operator HTML for a session.

Passthrough sessions get `proxy_intervention.html` (iframes the
substrate viewer + crops via crop_metrics); streaming sessions
get `intervention.html` (the screencast viewer).
Passthrough sessions get `passthrough-mode.html` (iframes the
substrate viewer + crops via crop_metrics); screencast sessions
get `screencast-mode.html` (the CDP-screencast viewer).
"""
session = self.sessions[session_id]
if session.is_passthrough:
template = jinja_env.get_template("proxy_intervention.html")
template = jinja_env.get_template("passthrough-mode.html")
return template.render(
access_token=session.access_token,
reason=reason,
Expand All @@ -1164,7 +1164,7 @@ def _get_html_client(self, session_id: str, reason: str) -> str:
stream_url=session.stream_url,
crop_metrics=session.crop_metrics,
)
template = jinja_env.get_template("intervention.html")
template = jinja_env.get_template("screencast-mode.html")
return template.render(
access_token=session.access_token,
reason=reason,
Expand All @@ -1187,18 +1187,6 @@ def get_operator_url(self, session_id: str) -> str:
token = self.sessions[session_id].access_token
return f"{base_url}/?t={token}"

def get_stream_url(self, session_id: str) -> str:
"""Deprecated alias for :meth:`get_operator_url`. Removed in v0.7."""
import warnings

warnings.warn(
"get_stream_url() is deprecated; use get_operator_url() instead. "
"Will be removed in v0.7.",
DeprecationWarning,
stacklevel=2,
)
return self.get_operator_url(session_id)

async def start(self) -> None:
"""Bind the port and serve until `stop()` is called."""
config = uvicorn.Config(
Expand Down
Loading