From a0670ccee3e011818fd2f2b447a858fbdc24924a Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 11:45:47 -0700 Subject: [PATCH 01/41] feat(cli): add pi as a hook-path coding agent Adds the pi coding agent to the NeMo Relay CLI as a hook-path agent, plus the pi extension that drives it. pi has no native hook-configuration file and its external stream is observation-only, so hook calls must originate inside an extension. The extension is a thin HTTP client to the gateway: it forwards pi's lifecycle to /hooks/pi and gates tool calls on the gateway's verdict. CLI side: - crates/cli/src/agents/pi/ with descriptor, adapter, launch and doctor - PiPayloadExtractor using SessionHeaderPolicy::RelayOnly so pi never inherits a stray x-claude-code-session-id - /hooks/pi route and pi_hook handler - Pi on CodingAgent, AgentKind, AgentArg and AgentConfigs pi has no plugin marketplace -- no `pi plugin` verb, no manifest, and no MCP client -- so the ~15 marketplace arms reject pi explicitly and point at `pi install ` and the auto-discovery directories, rather than synthesizing manifests pi will never read. Two edit sites the compiler does not enforce, both handled: - FileAgentsConfig carries deny_unknown_fields, so [agents.pi] needed the deserializer as well as the runtime struct - InstallTarget::All enumerates agents explicitly; pi is deliberately absent Extension side: - integrations/pi/ forwards session, agent-run, turn and tool lifecycle - tool_call is the only hook that awaits a verdict; the rest are fired without blocking pi's critical path and drained at session_shutdown - a guardrail rejection arrives as HTTP 403 with error.type = nemo_relay_guardrail_rejected, and error.reason is passed to pi verbatim, so the model reads the guardrail's own words Boundary choices worth noting: tool_execution_start is not forwarded as a tool start (it fires before validation and for calls that never execute), and tool_execution_end rather than tool_result is the end boundary (tool_result never fires for blocked calls). Tests: 2 Rust tests pin the 403 and 200 paths on /hooks/pi; 13 Node tests pin the extension's half of the contract. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- crates/cli/src/agents/mod.rs | 72 ++++- crates/cli/src/agents/pi/adapter.rs | 53 ++++ crates/cli/src/agents/pi/doctor.rs | 39 +++ crates/cli/src/agents/pi/launch.rs | 74 +++++ crates/cli/src/agents/pi/mod.rs | 52 ++++ crates/cli/src/agents/shared/adapters.rs | 27 ++ crates/cli/src/commands/root.rs | 2 + crates/cli/src/configuration/mod.rs | 4 + crates/cli/src/configuration/types.rs | 1 + crates/cli/src/events/mod.rs | 2 + crates/cli/src/server/mod.rs | 22 +- crates/cli/tests/cli_tests.rs | 5 +- .../coverage/agents/coding_agent_tests.rs | 13 +- crates/cli/tests/coverage/agents/pi_tests.rs | 34 +++ .../cli/tests/coverage/commands/main_tests.rs | 2 +- .../cli/tests/coverage/shared/server_tests.rs | 93 +++++++ integrations/pi/README.md | 119 ++++++++ integrations/pi/index.ts | 254 ++++++++++++++++++ integrations/pi/package.json | 17 ++ integrations/pi/src/gateway-client.ts | 154 +++++++++++ integrations/pi/src/pi-hook-types.ts | 119 ++++++++ .../pi/test/fixtures/reentry-driver.ts | 36 +++ integrations/pi/test/gateway-client.test.mjs | 205 ++++++++++++++ integrations/pi/tsconfig.json | 20 ++ 24 files changed, 1411 insertions(+), 8 deletions(-) create mode 100644 crates/cli/src/agents/pi/adapter.rs create mode 100644 crates/cli/src/agents/pi/doctor.rs create mode 100644 crates/cli/src/agents/pi/launch.rs create mode 100644 crates/cli/src/agents/pi/mod.rs create mode 100644 crates/cli/tests/coverage/agents/pi_tests.rs create mode 100644 integrations/pi/README.md create mode 100644 integrations/pi/index.ts create mode 100644 integrations/pi/package.json create mode 100644 integrations/pi/src/gateway-client.ts create mode 100644 integrations/pi/src/pi-hook-types.ts create mode 100644 integrations/pi/test/fixtures/reentry-driver.ts create mode 100644 integrations/pi/test/gateway-client.test.mjs create mode 100644 integrations/pi/tsconfig.json diff --git a/crates/cli/src/agents/mod.rs b/crates/cli/src/agents/mod.rs index 133b8ad98..9cdfae8ce 100644 --- a/crates/cli/src/agents/mod.rs +++ b/crates/cli/src/agents/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod claude; pub(crate) mod codex; +pub(crate) mod pi; pub(crate) mod shared; use semver::Version; @@ -15,6 +16,7 @@ pub(crate) enum CodingAgent { /// `claude-code` remains an input alias for older Relay configuration. ClaudeCode, Codex, + Pi, } #[derive(Debug, Clone, Copy)] @@ -29,13 +31,35 @@ pub(super) struct AgentDescriptor { hook_events: &'static [&'static str], } +/// Why pi has no marketplace-plugin implementation. +/// +/// Codex and Claude Code both install NeMo Relay through a plugin marketplace +/// (`codex plugin add`, `claude plugin install`) backed by a generated manifest. +/// pi has none of that: it has no `pi plugin` verb, no marketplace, and no MCP +/// client for a plugin-owned server to serve. Extensions are installed with +/// `pi install ` or by placing a file in an auto-discovered directory. +/// +/// Rather than synthesize manifests pi will never read, the marketplace surface +/// rejects pi explicitly and the gateway surface (hooks, launch, doctor) is +/// implemented for real. +pub(crate) const PI_MARKETPLACE_UNSUPPORTED: &str = "pi has no plugin marketplace; install the NeMo Relay pi extension with `pi install ` \ + or place it in `~/.pi/agent/extensions/`, then run pi through `nemo-relay launch pi`"; + +/// Reached only if a marketplace code path forgets to reject pi first. +macro_rules! pi_marketplace_unreachable { + () => { + unreachable!("{}", PI_MARKETPLACE_UNSUPPORTED) + }; +} + impl CodingAgent { - pub(crate) const ALL: [Self; 2] = [Self::ClaudeCode, Self::Codex]; + pub(crate) const ALL: [Self; 3] = [Self::ClaudeCode, Self::Codex, Self::Pi]; const fn descriptor(self) -> AgentDescriptor { match self { Self::ClaudeCode => claude::DESCRIPTOR, Self::Codex => codex::DESCRIPTOR, + Self::Pi => pi::DESCRIPTOR, } } @@ -108,6 +132,7 @@ impl CodingAgent { match self { Self::ClaudeCode => claude::parse_version(raw), Self::Codex => codex::parse_version(raw), + Self::Pi => pi::parse_version(raw), } } @@ -159,6 +184,7 @@ impl crate::installation::marketplace::MarketplaceHost for CodingAgent { match self { Self::Codex => &[".agents", "plugins", "marketplace.json"], Self::ClaudeCode => &[".claude-plugin", "marketplace.json"], + Self::Pi => pi_marketplace_unreachable!(), } } @@ -166,6 +192,7 @@ impl crate::installation::marketplace::MarketplaceHost for CodingAgent { match self { Self::Codex => &[".codex-plugin", "plugin.json"], Self::ClaudeCode => &[".claude-plugin", "plugin.json"], + Self::Pi => pi_marketplace_unreachable!(), } } @@ -206,6 +233,7 @@ impl crate::installation::marketplace::MarketplaceHost for CodingAgent { "--scope".into(), "user".into(), ], + Self::Pi => pi_marketplace_unreachable!(), } } @@ -213,6 +241,7 @@ impl crate::installation::marketplace::MarketplaceHost for CodingAgent { match self { Self::Codex => vec!["plugin".into(), "remove".into(), plugin_id.into()], Self::ClaudeCode => vec!["plugin".into(), "uninstall".into(), plugin_name.into()], + Self::Pi => pi_marketplace_unreachable!(), } } @@ -228,6 +257,7 @@ impl crate::installation::marketplace::MarketplaceHost for CodingAgent { Self::ClaudeCode => { crate::installation::marketplace::host::claude_registration_report(options, runner) } + Self::Pi => Err(PI_MARKETPLACE_UNSUPPORTED.to_string()), } } @@ -243,6 +273,7 @@ impl crate::installation::marketplace::MarketplaceHost for CodingAgent { Self::ClaudeCode => format!( "cannot safely replace or uninstall an existing Claude Code plugin because its MCP generation marker {problem}; close all Claude Code clients and standalone `nemo-relay mcp` processes, run `claude plugin uninstall nemo-relay-plugin` and `claude plugin marketplace remove nemo-relay-local`, remove the stale marketplace and state from the selected install directory, then run `nemo-relay install claude-code --force` to create a fenced install (and `nemo-relay uninstall claude-code` afterward if removal was intended)" ), + Self::Pi => pi_marketplace_unreachable!(), } } @@ -268,6 +299,8 @@ impl crate::installation::marketplace::MarketplaceHost for CodingAgent { || plugin_root.join(".mcp.json").exists() || generation_fence.exists() } + // pi installs nothing through this surface, so nothing to detect. + Self::Pi => false, } } @@ -329,6 +362,7 @@ pub(crate) fn marketplace_manifest( match agent { CodingAgent::Codex => codex::assets::marketplace_manifest(marketplace, plugin), CodingAgent::ClaudeCode => claude::assets::marketplace_manifest(marketplace, plugin), + CodingAgent::Pi => pi_marketplace_unreachable!(), } } @@ -336,6 +370,7 @@ pub(crate) fn plugin_manifest(agent: CodingAgent, plugin: &str) -> serde_json::V match agent { CodingAgent::Codex => codex::assets::plugin_manifest(plugin), CodingAgent::ClaudeCode => claude::assets::plugin_manifest(plugin), + CodingAgent::Pi => pi_marketplace_unreachable!(), } } @@ -346,6 +381,8 @@ pub(crate) fn plugin_mcp_config( match agent { CodingAgent::Codex => codex::assets::mcp_config(server), CodingAgent::ClaudeCode => Ok(claude::assets::mcp_config(server)), + // pi ships no MCP client, so a plugin-owned server would have no consumer. + CodingAgent::Pi => Err(PI_MARKETPLACE_UNSUPPORTED.to_string()), } } @@ -374,6 +411,7 @@ pub(crate) fn prepare_launch( CodingAgent::ClaudeCode => { claude::launch::prepare(launch, gateway_url, proxy_credential, dry_run) } + CodingAgent::Pi => pi::launch::prepare(launch, gateway_url), } } @@ -388,6 +426,7 @@ pub(crate) const fn config( match agent { CodingAgent::ClaudeCode => &configs.claude, CodingAgent::Codex => &configs.codex, + CodingAgent::Pi => &configs.pi, } } @@ -398,6 +437,7 @@ pub(crate) fn hook_status( match agent { CodingAgent::Codex => codex::doctor::hook_status(), CodingAgent::ClaudeCode => claude::doctor::hook_status(), + CodingAgent::Pi => pi::doctor::hook_status(), } } @@ -428,6 +468,7 @@ pub(crate) fn snapshot_setup(agent: CodingAgent) -> Result snapshot_codex_setup().map(SetupSnapshot::Codex), CodingAgent::ClaudeCode => snapshot_claude_setup().map(SetupSnapshot::Claude), + CodingAgent::Pi => Err(PI_MARKETPLACE_UNSUPPORTED.to_string()), } } @@ -449,6 +490,7 @@ pub(crate) fn setup_marketplace_plugin( install_codex_plugin_with_generation(gateway_url, plugin_root, generation_token) } CodingAgent::ClaudeCode => enable_claude_provider(gateway_url), + CodingAgent::Pi => Err(PI_MARKETPLACE_UNSUPPORTED.to_string()), } } @@ -460,6 +502,7 @@ pub(crate) fn uninstall_marketplace_plugin( match agent { CodingAgent::Codex => uninstall_codex_plugin(gateway_url, plugin_root), CodingAgent::ClaudeCode => restore_claude_provider(gateway_url), + CodingAgent::Pi => Err(PI_MARKETPLACE_UNSUPPORTED.to_string()), } } @@ -477,6 +520,7 @@ pub(crate) fn doctor_marketplace_plugin( generation_token, ), CodingAgent::ClaudeCode => doctor_plugin(CodingAgent::ClaudeCode, gateway_url, plugin_root), + CodingAgent::Pi => Err(PI_MARKETPLACE_UNSUPPORTED.to_string()), } } @@ -490,6 +534,7 @@ pub(crate) fn doctor_marketplace_plugin_json( CodingAgent::ClaudeCode => { doctor_plugin_json(CodingAgent::ClaudeCode, gateway_url, plugin_root) } + CodingAgent::Pi => Err(PI_MARKETPLACE_UNSUPPORTED.to_string()), } } @@ -500,6 +545,9 @@ pub(crate) fn install_integration( match agent { CodingAgent::Codex => codex::install::install(command), CodingAgent::ClaudeCode => claude::install::install(command), + CodingAgent::Pi => Err(crate::error::CliError::Install( + PI_MARKETPLACE_UNSUPPORTED.to_string(), + )), } } @@ -510,6 +558,9 @@ pub(crate) fn uninstall_integration( match agent { CodingAgent::Codex => codex::install::uninstall(command), CodingAgent::ClaudeCode => claude::install::uninstall(command), + CodingAgent::Pi => Err(crate::error::CliError::Install( + PI_MARKETPLACE_UNSUPPORTED.to_string(), + )), } } @@ -789,6 +840,21 @@ pub(crate) fn doctor_plugin_json( Some(trust), ) } + // pi's hooks live inside a user-loaded extension rather than in a + // NeMo Relay-managed plugin root, so the only checkable fact here is + // whether that extension is discoverable. + CodingAgent::Pi => { + let extension = pi::doctor::extension_configured(); + ( + json!({ + "plugin_binary": plugin_binary, + "sidecar_running": sidecar_running, + "pi_extension_located": extension + }), + plugin_binary && extension, + None, + ) + } }; let mut report = json!({ "ok": ok, @@ -848,6 +914,10 @@ fn doctor_ok( print_info("codex hook trust", &trust.summary()); } } + CodingAgent::Pi => { + ok &= print_check("pi extension located", pi::doctor::extension_configured()); + print_info("pi hooks", &pi::doctor::hook_status()?); + } } Ok(ok) } diff --git a/crates/cli/src/agents/pi/adapter.rs b/crates/cli/src/agents/pi/adapter.rs new file mode 100644 index 000000000..ccaab7f14 --- /dev/null +++ b/crates/cli/src/agents/pi/adapter.rs @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use axum::http::HeaderMap; +use serde_json::{Value, json}; + +use crate::agents::shared::adapters::{ + AdapterOutcome, ClassificationRules, PI_PAYLOAD_EXTRACTOR, classify, +}; +use crate::events::AgentKind; + +/// Normalizes pi extension hook payloads and returns the response the extension expects. +/// +/// The response body is deliberately minimal. A tool call is *allowed* by this +/// endpoint returning 2xx; it is *blocked* by `apply_events` failing the +/// conditional-execution guardrail chain, which surfaces as HTTP 403 with +/// `error.type = "nemo_relay_guardrail_rejected"` and the guardrail's own words +/// in `error.reason`. The extension turns that into pi's `{block, reason}`, +/// which pi passes verbatim to the model as an error tool result. +/// +/// Mapping notes: +/// - pi's `session_start`/`session_shutdown` are the session boundary, not +/// `agent_start`/`agent_end`. One pi session can re-enter the agent run many +/// times (provider retry, compaction, queued follow-up), so treating +/// `agent_start` as a session start would open a session per retry. +/// - `agent_settled` is the only pi event that fires exactly once per logical +/// agent run, so it is the turn-boundary snapshot rather than `agent_end`. +/// - `tool_call` is the gating hook and maps to tool start. `tool_execution_start` +/// is deliberately NOT mapped: it fires before validation and before +/// `tool_call`, including for calls that never execute, so using it to open a +/// tool span would create spans for calls pi then discards. +pub(crate) fn adapt(payload: Value, headers: &HeaderMap) -> AdapterOutcome { + let events = classify( + &payload, + headers, + &PI_PAYLOAD_EXTRACTOR, + &ClassificationRules { + kind: AgentKind::Pi, + agent_start: &["session_start", "sessionStart"], + agent_end: &["session_shutdown", "sessionShutdown"], + // pi ships no MCP client and has no nested-agent hook of its own; + // subagents are an extension-level concept it does not surface. + subagent_start: &[], + subagent_end: &[], + tool_start: &["tool_call", "toolCall"], + tool_end: &["tool_execution_end", "toolExecutionEnd"], + }, + ); + AdapterOutcome { + events, + response: json!({}), + } +} diff --git a/crates/cli/src/agents/pi/doctor.rs b/crates/cli/src/agents/pi/doctor.rs new file mode 100644 index 000000000..19a0ccf5e --- /dev/null +++ b/crates/cli/src/agents/pi/doctor.rs @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Diagnostics for the pi integration. +//! +//! Codex and Claude Code can be checked by inspecting files NeMo Relay wrote +//! (generated hook config, a settings base URL). pi's hooks live inside an +//! extension the user loads, so the only thing checkable from here is whether +//! that extension is discoverable. + +use std::path::PathBuf; + +use super::launch::PI_EXTENSION_PATH_ENV; + +/// Human-readable hook status for `nemo-relay doctor`. +pub(crate) fn hook_status() -> Result { + match extension_location() { + Some(path) => Ok(format!( + "pi extension resolved at {} (hooks are emitted by the extension, not by pi itself)", + path.display() + )), + None => Ok(format!( + "pi extension not located; set {PI_EXTENSION_PATH_ENV}, or install the extension with \ + `pi install ` or into an auto-discovered directory \ + (`~/.pi/agent/extensions/`, `.pi/extensions/`)" + )), + } +} + +/// Whether the extension entry point can be found. +pub(crate) fn extension_configured() -> bool { + extension_location().is_some() +} + +fn extension_location() -> Option { + std::env::var_os(PI_EXTENSION_PATH_ENV) + .map(PathBuf::from) + .filter(|path| path.exists()) +} diff --git a/crates/cli/src/agents/pi/launch.rs b/crates/cli/src/agents/pi/launch.rs new file mode 100644 index 000000000..b7325444d --- /dev/null +++ b/crates/cli/src/agents/pi/launch.rs @@ -0,0 +1,74 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Transparent launch for pi. +//! +//! pi differs from Codex and Claude Code in two ways that shape this module. +//! +//! **Hooks cannot be injected from outside.** Codex takes `--config hooks.*=...` +//! and Claude Code reads a settings file, so their launchers can install hook +//! commands directly. pi has no native hook-configuration file and its external +//! stream is observation-only, so hook calls must originate inside an extension. +//! Launch therefore loads the NeMo Relay extension with `-e` and passes the +//! gateway URL through the environment for it to read. +//! +//! **Model traffic cannot be redirected by a flag or a generic env var.** pi +//! resolves `baseUrl` per model from its generated catalog; the only documented +//! override points are per-provider (`AZURE_OPENAI_BASE_URL`, `LLAMA_BASE_URL`) +//! or a provider registered by an extension. So redirection is the extension's +//! job too, and this module only supplies the URL. + +use std::path::PathBuf; + +use crate::error::CliError; +use crate::process::{PreparedAgentLaunch, insert_after_host}; + +/// Environment variable the pi extension reads to find the gateway. +pub(crate) const PI_GATEWAY_URL_ENV: &str = "NEMO_RELAY_PI_GATEWAY_URL"; + +/// Environment variable pointing pi at the NeMo Relay extension entry point. +pub(crate) const PI_EXTENSION_PATH_ENV: &str = "NEMO_RELAY_PI_EXTENSION"; + +pub(crate) fn prepare(launch: &mut PreparedAgentLaunch, gateway_url: &str) -> Result<(), CliError> { + set_env(launch, PI_GATEWAY_URL_ENV, gateway_url); + + // `-e` is the right loader here: it is trust-ungated, loads before + // discovery, and survives `--no-extensions`, so a launched session gets the + // extension regardless of the user's own pi configuration. + let Some(path) = extension_path() else { + return Err(CliError::Launch(format!( + "could not locate the NeMo Relay pi extension; set {PI_EXTENSION_PATH_ENV} to its \ + entry point, or install it with `pi install ` and launch pi directly" + ))); + }; + let rendered = path.display().to_string(); + set_env(launch, PI_EXTENSION_PATH_ENV, &rendered); + insert_after_host( + &mut launch.argv, + launch.host_index, + ["-e".to_string(), rendered], + ); + + launch.notes.push( + "pi model traffic is redirected by the NeMo Relay extension registering a gateway-backed \ + provider; pi has no base-URL flag or generic environment override" + .to_string(), + ); + Ok(()) +} + +fn set_env(launch: &mut PreparedAgentLaunch, name: &str, value: &str) { + launch.env.retain(|(existing, _)| existing != name); + launch.env.push((name.to_string(), value.to_string())); +} + +/// Resolve the extension entry point, preferring an explicit override. +/// +/// There is no installed location to fall back on the way Codex and Claude Code +/// have one, because pi extensions live in the user's own configuration +/// directories rather than in a NeMo Relay-managed plugin root. +fn extension_path() -> Option { + std::env::var_os(PI_EXTENSION_PATH_ENV) + .map(PathBuf::from) + .filter(|path| path.exists()) +} diff --git a/crates/cli/src/agents/pi/mod.rs b/crates/cli/src/agents/pi/mod.rs new file mode 100644 index 000000000..5e841f0f8 --- /dev/null +++ b/crates/cli/src/agents/pi/mod.rs @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! pi coding-agent identity and compatibility policy. +//! +//! pi has no native hook-configuration file and its external stream is +//! observation-only, so hook calls originate inside a pi *extension* that posts +//! to `/hooks/pi`. The extension is the only component that can gate a tool call +//! before it runs, which is why the hook event names below are pi's own +//! extension hook names rather than the `PreToolUse`/`PostToolUse` vocabulary +//! Codex and Claude Code use. + +use semver::Version; + +use super::AgentDescriptor; + +pub(crate) mod doctor; +pub(crate) mod launch; + +pub(super) const DESCRIPTOR: AgentDescriptor = AgentDescriptor { + argument: "pi", + install_argument: "pi", + label: "pi", + executable: "pi", + hook_path: "/hooks/pi", + version_product: "pi", + // pi ships breaking changes through minor releases and has no major-release + // channel, so this floor is the version the integration was verified + // against rather than a lower bound that is expected to keep holding. + minimum_version: (0, 84, 0), + hook_events: &[ + "session_start", + "session_shutdown", + "agent_start", + "agent_end", + "agent_settled", + "turn_start", + "turn_end", + "tool_call", + "tool_execution_start", + "tool_execution_end", + ], +}; + +/// `pi --version` prints a bare semver line with no product prefix. +pub(super) fn parse_version(raw: &str) -> Option { + Version::parse(raw.trim()).ok() +} + +#[cfg(test)] +#[path = "../../../tests/coverage/agents/pi_tests.rs"] +mod tests; diff --git a/crates/cli/src/agents/shared/adapters.rs b/crates/cli/src/agents/shared/adapters.rs index 8d5b3aff5..30e970a3c 100644 --- a/crates/cli/src/agents/shared/adapters.rs +++ b/crates/cli/src/agents/shared/adapters.rs @@ -5,6 +5,8 @@ pub(crate) mod claude_code; #[path = "../codex/adapter.rs"] pub(crate) mod codex; +#[path = "../pi/adapter.rs"] +pub(crate) mod pi; pub(crate) const SKILL_LOAD_SOURCE_KEY: &str = "skill_load_source"; pub(crate) const SKILL_LOAD_SOURCE_PROMPT_EXPANSION: &str = "prompt_expansion"; @@ -182,10 +184,26 @@ pub(crate) trait AgentPayloadExtractor { pub(super) struct ClaudeCodePayloadExtractor; pub(super) struct CodexPayloadExtractor; +pub(super) struct PiPayloadExtractor; pub(super) static CLAUDE_CODE_PAYLOAD_EXTRACTOR: ClaudeCodePayloadExtractor = ClaudeCodePayloadExtractor; pub(super) static CODEX_PAYLOAD_EXTRACTOR: CodexPayloadExtractor = CodexPayloadExtractor; +pub(super) static PI_PAYLOAD_EXTRACTOR: PiPayloadExtractor = PiPayloadExtractor; + +/// pi hooks are emitted by a NeMo Relay-authored extension, so the payload uses +/// the canonical key names and needs no path deviations. The one override is the +/// session-header policy: pi is not Claude Code installed mode and must not +/// adopt an `x-claude-code-session-id` that happens to be in the environment. +impl AgentPayloadExtractor for PiPayloadExtractor { + fn session_header_policy(&self) -> SessionHeaderPolicy { + SessionHeaderPolicy::RelayOnly + } + + fn tool_paths(&self) -> &'static ToolPathSet { + PI_TOOL_PATHS + } +} /// Claude Code reports its native tool identifier as `tool_use_id`, so it uses /// a tool path set that prefers that key. Every other hook field matches the @@ -367,6 +385,15 @@ const CODEX_TOOL_PATHS: &ToolPathSet = &ToolPathSet { result: TOOL_RESULT_PATHS, status: TOOL_STATUS_PATHS, }; +/// pi sends its native `tool_call` arguments under `input`, which the shared +/// argument precedence already covers ahead of `arguments`/`args`. +const PI_TOOL_PATHS: &ToolPathSet = &ToolPathSet { + call_id: TOOL_CALL_ID_PATHS, + name: TOOL_NAME_PATHS, + arguments: TOOL_ARGUMENT_PATHS, + result: TOOL_RESULT_PATHS, + status: TOOL_STATUS_PATHS, +}; fn agent_session_id( headers: &HeaderMap, diff --git a/crates/cli/src/commands/root.rs b/crates/cli/src/commands/root.rs index 79e9ffaf4..8692f9158 100644 --- a/crates/cli/src/commands/root.rs +++ b/crates/cli/src/commands/root.rs @@ -21,6 +21,7 @@ pub(crate) enum AgentArg { #[value(name = "claude", alias = "claude-code")] Claude, Codex, + Pi, } impl From for CodingAgent { @@ -28,6 +29,7 @@ impl From for CodingAgent { match value { AgentArg::Claude => Self::ClaudeCode, AgentArg::Codex => Self::Codex, + AgentArg::Pi => Self::Pi, } } } diff --git a/crates/cli/src/configuration/mod.rs b/crates/cli/src/configuration/mod.rs index 44f7ceff1..f1ed9f8da 100644 --- a/crates/cli/src/configuration/mod.rs +++ b/crates/cli/src/configuration/mod.rs @@ -81,6 +81,7 @@ struct FileAgentsConfig { // `CodingAgent` enum kebab spelling. Same convention as the bare-agent shortcut in Phase 2. claude: Option, codex: Option, + pi: Option, } #[derive(Debug, Clone, Default, Deserialize)] @@ -1569,6 +1570,9 @@ fn apply_file_agents_config(agents: &mut AgentConfigs, file_agents: Option "codex", Self::ClaudeCode => "claude-code", + Self::Pi => "pi", Self::Gateway => "gateway", } } diff --git a/crates/cli/src/server/mod.rs b/crates/cli/src/server/mod.rs index 157a0352d..678e59f9f 100644 --- a/crates/cli/src/server/mod.rs +++ b/crates/cli/src/server/mod.rs @@ -41,7 +41,7 @@ use subtle::ConstantTimeEq; use tokio::net::TcpListener; use tokio::sync::oneshot; -use crate::agents::shared::adapters::{claude_code, codex}; +use crate::agents::shared::adapters::{claude_code, codex, pi}; use crate::configuration::{ BOOTSTRAP_CLIENT_TOKEN_HEADER, BootstrapChallengeKey, GatewayConfig, ManagedBootstrapIdentity, }; @@ -586,6 +586,7 @@ fn router_with_state(state: AppState) -> Router { .route("/bootstrap/shutdown", post(shutdown_bootstrap_sidecar)) .route("/hooks/codex", post(codex_hook)) .route("/hooks/claude-code", post(claude_code_hook)) + .route("/hooks/pi", post(pi_hook)) .route("/responses", post(gateway::passthrough)) .route("/chat/completions", post(gateway::passthrough)) .route("/models", get(gateway::models)) @@ -1207,6 +1208,25 @@ async fn claude_code_hook( Ok(Json(outcome.response)) } +// Handles pi extension hooks. pi has no native hook-config file, so these arrive from a NeMo +// Relay-authored extension rather than from pi itself. Events are committed before the response +// so a conditional-execution guardrail rejection surfaces as HTTP 403 and the extension can turn +// it into pi's `{block, reason}` before the tool runs. +async fn pi_hook( + State(state): State, + headers: HeaderMap, + payload: Result, JsonRejection>, +) -> Result, CliError> { + state.touch(); + let Json(payload) = payload.map_err(hook_payload_rejection)?; + let outcome = pi::adapt(payload, &headers); + state + .sessions + .apply_events(&headers, outcome.events) + .await?; + Ok(Json(outcome.response)) +} + fn hook_payload_rejection(rejection: JsonRejection) -> CliError { if rejection.status() == axum::http::StatusCode::PAYLOAD_TOO_LARGE { CliError::PayloadTooLarge(rejection.to_string()) diff --git a/crates/cli/tests/cli_tests.rs b/crates/cli/tests/cli_tests.rs index 8f107848a..07ada027f 100644 --- a/crates/cli/tests/cli_tests.rs +++ b/crates/cli/tests/cli_tests.rs @@ -1792,7 +1792,10 @@ fn cli_agents_json_emits_supported_agent_shapes() { .iter() .map(|agent| agent["name"].as_str().unwrap()) .collect::>(); - assert_eq!(names, std::collections::BTreeSet::from(["claude", "codex"])); + assert_eq!( + names, + std::collections::BTreeSet::from(["claude", "codex", "pi"]) + ); assert!(agents.iter().all(|agent| agent["status"].is_string())); } diff --git a/crates/cli/tests/coverage/agents/coding_agent_tests.rs b/crates/cli/tests/coverage/agents/coding_agent_tests.rs index 63538f2ba..e94f78590 100644 --- a/crates/cli/tests/coverage/agents/coding_agent_tests.rs +++ b/crates/cli/tests/coverage/agents/coding_agent_tests.rs @@ -10,14 +10,19 @@ fn agent_descriptors_are_complete_and_unique() { let executables = CodingAgent::ALL.map(CodingAgent::executable); let hook_paths = CodingAgent::ALL.map(CodingAgent::hook_path); - assert_eq!(arguments, ["claude", "codex"]); - assert_eq!(install_arguments, ["claude-code", "codex"]); - assert_eq!(executables, ["claude", "codex"]); - assert_eq!(hook_paths, ["/hooks/claude-code", "/hooks/codex"]); + assert_eq!(arguments, ["claude", "codex", "pi"]); + assert_eq!(install_arguments, ["claude-code", "codex", "pi"]); + assert_eq!(executables, ["claude", "codex", "pi"]); + assert_eq!( + hook_paths, + ["/hooks/claude-code", "/hooks/codex", "/hooks/pi"] + ); assert_eq!(CodingAgent::ClaudeCode.label(), "Claude Code"); assert_eq!(CodingAgent::Codex.label(), "Codex"); assert_eq!(CodingAgent::ClaudeCode.hook_events().len(), 14); assert_eq!(CodingAgent::Codex.hook_events().len(), 10); + assert_eq!(CodingAgent::Pi.label(), "pi"); + assert_eq!(CodingAgent::Pi.hook_events().len(), 10); for agent in CodingAgent::ALL { let events = agent.hook_events(); assert!(events.iter().all(|event| !event.is_empty())); diff --git a/crates/cli/tests/coverage/agents/pi_tests.rs b/crates/cli/tests/coverage/agents/pi_tests.rs new file mode 100644 index 000000000..c09e4e1ac --- /dev/null +++ b/crates/cli/tests/coverage/agents/pi_tests.rs @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; + +#[test] +fn parses_bare_semver_version_output() { + // `pi --version` prints only the version, with no product prefix, unlike + // `codex-cli 0.143.0` or `2.1.121 (Claude Code)`. + assert_eq!(parse_version("0.84.0"), Some(Version::new(0, 84, 0))); + assert_eq!(parse_version(" 0.84.0 "), Some(Version::new(0, 84, 0))); +} + +#[test] +fn rejects_prefixed_or_empty_version_output() { + assert_eq!(parse_version("pi 0.84.0"), None); + assert_eq!(parse_version(""), None); + assert_eq!(parse_version("not-a-version"), None); +} + +#[test] +fn descriptor_routes_to_the_pi_hook_endpoint() { + assert_eq!(DESCRIPTOR.hook_path, "/hooks/pi"); + assert_eq!(DESCRIPTOR.executable, "pi"); +} + +#[test] +fn hook_events_use_pi_vocabulary_not_codex_vocabulary() { + // pi hooks originate in a NeMo Relay-authored extension, so the descriptor + // lists pi's own hook names rather than PreToolUse/PostToolUse. + assert!(DESCRIPTOR.hook_events.contains(&"tool_call")); + assert!(DESCRIPTOR.hook_events.contains(&"agent_settled")); + assert!(!DESCRIPTOR.hook_events.contains(&"PreToolUse")); +} diff --git a/crates/cli/tests/coverage/commands/main_tests.rs b/crates/cli/tests/coverage/commands/main_tests.rs index f0b9f2b86..a1d7cf9be 100644 --- a/crates/cli/tests/coverage/commands/main_tests.rs +++ b/crates/cli/tests/coverage/commands/main_tests.rs @@ -384,7 +384,7 @@ fn multi_agent_operations_attempt_every_target_before_reporting_errors() { visited.borrow_mut().push(agent); match agent { CodingAgent::Codex => Err(error::CliError::Install("codex failure".into())), - CodingAgent::ClaudeCode => Ok(ExitCode::FAILURE), + CodingAgent::ClaudeCode | CodingAgent::Pi => Ok(ExitCode::FAILURE), } }) .unwrap_err() diff --git a/crates/cli/tests/coverage/shared/server_tests.rs b/crates/cli/tests/coverage/shared/server_tests.rs index ff058afe0..78968b77b 100644 --- a/crates/cli/tests/coverage/shared/server_tests.rs +++ b/crates/cli/tests/coverage/shared/server_tests.rs @@ -2281,6 +2281,99 @@ async fn pre_tool_hook_rejects_when_conditional_guardrail_blocks() { ); assert_eq!(body["error"]["reason"], json!("blocked by policy")); } + +// pi's extension gates a tool call on this endpoint's verdict, so the 403 shape +// is a wire contract, not an internal detail: the extension turns +// `error.reason` into pi's `{block, reason}`, which pi hands to the model +// verbatim as an error tool result. +#[tokio::test] +async fn pi_tool_call_hook_rejects_when_conditional_guardrail_blocks() { + let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; + let _ = deregister_tool_conditional_execution_guardrail("cli-pi-tool-blocker"); + const BLOCKED_TEST_TOOL: &str = "read"; + register_tool_conditional_execution_guardrail( + "cli-pi-tool-blocker", + 1, + Arc::new(|name, args| { + Box::pin(async move { + let targets_secret = args + .get("path") + .and_then(Value::as_str) + .is_some_and(|path| path.ends_with(".env")); + Ok((name == BLOCKED_TEST_TOOL && targets_secret) + .then(|| "read .env is blocked; use .env.example".to_string())) + }) + }), + ) + .unwrap(); + let _cleanup = ToolGuardrailCleanup("cli-pi-tool-blocker"); + + let app = router(test_config()); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/hooks/pi") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "session_id": "pi-guardrail-session", + "hook_event_name": "tool_call", + "tool_call_id": "call-1", + "tool_name": BLOCKED_TEST_TOOL, + "input": { "path": "/work/.env" } + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + body["error"]["type"], + json!("nemo_relay_guardrail_rejected") + ); + // The reason must survive verbatim: it is what the model reads. + assert_eq!( + body["error"]["reason"], + json!("read .env is blocked; use .env.example") + ); +} + +// The same endpoint must stay out of the way when no guardrail objects, +// otherwise every pi tool call would be blocked by a fail-closed extension. +#[tokio::test] +async fn pi_tool_call_hook_allows_when_no_guardrail_objects() { + let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; + let app = router(test_config()); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/hooks/pi") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "session_id": "pi-allow-session", + "hook_event_name": "tool_call", + "tool_call_id": "call-2", + "tool_name": "read", + "input": { "path": "/work/README.md" } + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); +} + #[tokio::test] async fn gateway_forwards_openai_json_without_rewriting_payload() { let upstream = spawn_upstream(false).await; diff --git a/integrations/pi/README.md b/integrations/pi/README.md new file mode 100644 index 000000000..8ac982194 --- /dev/null +++ b/integrations/pi/README.md @@ -0,0 +1,119 @@ + + +# NeMo Relay extension for pi + +A pi extension that forwards pi's lifecycle to the NeMo Relay CLI gateway and +gates tool calls on the gateway's verdict. + +pi has no native hook-configuration file, and its external stream is +observation-only, so hook calls have to originate inside an extension. Unlike +the Codex and Claude Code integrations — which install hook commands the host +runs for them — this extension *is* the hook client. It is deliberately thin: +all policy and all span construction happen in the gateway. + +## Status + +Proof of concept, tracked under +[RELAY-727](https://linear.app/nvidia/issue/RELAY-727) and +[RELAY-728](https://linear.app/nvidia/issue/RELAY-728). Verified against pi +`v0.84.0`. pi ships breaking changes through *minor* releases and has no +major-release channel, so re-verify hook signatures before relying on them. + +## Usage + +Start the gateway, then load the extension: + +```bash +nemo-relay --bind 127.0.0.1:4040 & +NEMO_RELAY_PI_GATEWAY_URL=http://127.0.0.1:4040 \ + pi -e integrations/pi/index.ts +``` + +`pi -e` is trust-ungated, loads before discovery, and survives +`--no-extensions`, which makes it the reliable way to load this. For everyday +use, install it with `pi install ` or place it in an auto-discovered +directory (`~/.pi/agent/extensions/`, `.pi/extensions/`). + +### Environment + +| Variable | Default | Meaning | +|---|---|---| +| `NEMO_RELAY_PI_GATEWAY_URL` | `http://127.0.0.1:4040` | Gateway base URL | +| `NEMO_RELAY_PI_TIMEOUT_MS` | `5000` | Per-request timeout | +| `NEMO_RELAY_PI_FAIL` | `open` | `closed` blocks tool calls when the gateway is unreachable | + +## How tool gating works + +`tool_call` is the only pi hook that can block, and for model-invoked tools it +is the only pre-execution decision point that sees arguments — pi's `--tools`, +`--exclude-tools`, `--no-tools` and runtime `setActiveTools` are all applied at +tool-registry construction, never per call. + +The wire contract, pinned from both sides by tests: + +| Gateway response | Extension behaviour | +|---|---| +| 2xx | allow | +| 403 with `error.type = "nemo_relay_guardrail_rejected"` | block, using `error.reason` | +| 403 without that marker | fault — an authorization failure is not a policy decision | +| other status, timeout, unreachable | fault, resolved by `NEMO_RELAY_PI_FAIL` | + +The block reason reaches the model **verbatim**: pi hands it to +`createErrorToolResult` with no framing. Write guardrail reasons as guidance, not +as error codes — a reason that says what to do instead produces a model that +adapts rather than one that gives up. + +## Hook mapping + +pi's lifecycle is `session -> agent run -> turn -> message | tool execution`. +Two shapes make a naive mapping wrong. + +**Agent-run re-entry.** One prompt can re-enter the agent run several times +(provider retry, post-compaction, queued follow-up), and pi's `turnIndex` resets +to 0 each time. The extension-facing `agent_end` carries no `willRetry` marker, +so a retry cannot be detected there; `agent_settled` is the only event that fires +exactly once per logical run. The gateway's own model is flat +(session -> turn -> tool) and assigns its own monotonic turn index, so the +extension sends `attempt_index` and a session-monotonic `turn_seq` as metadata — +they are the only way to recover which attempt a turn belonged to. + +**Concurrent tools.** pi preflights sibling calls sequentially then executes them +concurrently, so `tool_execution_end` arrives out of submission order. All +per-call state is keyed by `toolCallId`, the only correlator pi provides. + +| pi hook | Forwarded as | Note | +|---|---|---| +| `session_start` / `session_shutdown` | session boundary | **Not** `agent_start`/`agent_end` — those repeat on re-entry | +| `agent_start` / `agent_end` | attempt markers | Carry `attempt_index`; not a run boundary | +| `agent_settled` | logical run boundary | Fires exactly once, from a `finally` | +| `turn_start` / `turn_end` | turn boundary | Carries `turn_index` **and** `turn_seq` | +| `tool_call` | tool start, and the gate | The only blocking hook | +| `tool_execution_end` | tool end | For **every** outcome, including blocked | +| `tool_execution_start` | *not forwarded* | Fires before validation and for calls that never execute | + +`tool_result` is deliberately unused: it does not fire for blocked calls, and in +the parallel path it fires *before* `tool_execution_end`. + +## Development + +```bash +npm run typecheck --prefix integrations/pi +node --test integrations/pi/test/*.test.mjs +``` + +The gateway half of the contract is covered in Rust by +`pi_tool_call_hook_rejects_when_conditional_guardrail_blocks` and +`pi_tool_call_hook_allows_when_no_guardrail_objects` in +`crates/cli/tests/coverage/shared/server_tests.rs`. + +`test/fixtures/reentry-driver.ts` forces exactly one agent-run re-entry through +pi's real queued-follow-up path, for reproducing the colliding-turn-index case. + +## Related + +- CLI agent definition: `crates/cli/src/agents/pi/` +- Hook route: `/hooks/pi` in `crates/cli/src/server/mod.rs` +- Payload classification: `PiPayloadExtractor` in `crates/cli/src/agents/shared/adapters.rs` diff --git a/integrations/pi/index.ts b/integrations/pi/index.ts new file mode 100644 index 000000000..13e1a3abc --- /dev/null +++ b/integrations/pi/index.ts @@ -0,0 +1,254 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * NeMo Relay extension for the pi coding agent. + * + * pi has no native hook-configuration file and its external stream is + * observation-only, so hook calls must originate inside an extension. This one + * is a thin HTTP client to the NeMo Relay CLI gateway: it forwards pi's + * lifecycle to `/hooks/pi`, and gates tool calls on the gateway's verdict. + * + * **Governance.** `tool_call` is the only pi hook that can block, and for + * model-invoked tools it is the only pre-execution decision point that sees + * arguments -- `--tools` / `--exclude-tools` / `--no-tools` and the runtime + * `setActiveTools` are applied at tool-registry construction, never per call. + * A gateway guardrail rejection arrives as HTTP 403 and is translated into + * `{block, reason}`; pi hands that reason to the model verbatim, so the model + * reads the guardrail's own words. + * + * **Lifecycle mapping.** Two pi shapes make a naive mapping wrong: + * + * 1. *Agent-run re-entry.* One prompt can re-enter the agent run several times + * (provider retry, post-compaction, queued follow-up), and `turnIndex` resets + * to 0 each time, so turn indices collide within one prompt. The extension + * `agent_end` payload carries no `willRetry` marker, so a retry cannot be + * detected there. `agent_settled` is the only event that fires exactly once + * per logical run. Both an attempt counter and a session-monotonic turn + * sequence are therefore sent as metadata, because the gateway's own model is + * flat (session -> turn -> tool) and cannot express the nesting. + * 2. *Concurrent tools.* pi preflights sibling calls sequentially then executes + * them concurrently, so `tool_execution_end` arrives out of submission order. + * All per-call state is keyed by `toolCallId`, which is the only correlator + * pi provides. + * + * Load it with `pi -e `, or let `nemo-relay launch pi` do it. + * + * Environment (set by the launcher, overridable by hand): + * - `NEMO_RELAY_PI_GATEWAY_URL` gateway base URL (default `http://127.0.0.1:4040`) + * - `NEMO_RELAY_PI_TIMEOUT_MS` per-request timeout (default 5000) + * - `NEMO_RELAY_PI_FAIL` `closed` to block when the gateway is unreachable + */ +import { + type GatewayConfig, + configFromEnv, + postAndForget, + postHook, + resolveFault, +} from './src/gateway-client.ts'; +import type { + AgentEndEvent, + AgentSettledEvent, + AgentStartEvent, + ExtensionAPI, + ExtensionContext, + SessionShutdownEvent, + SessionStartEvent, + ToolCallEvent, + ToolCallEventResult, + ToolExecutionEndEvent, + ToolExecutionStartEvent, + TurnEndEvent, + TurnStartEvent, +} from './src/pi-hook-types.ts'; + +export default function nemoRelayExtension(pi: ExtensionAPI): void { + let config: GatewayConfig | null = null; + + /** Attempt counter within one logical agent run; reset on `agent_settled`. */ + let attemptIndex = 0; + /** Session-monotonic turn counter; pi's own `turnIndex` resets on re-entry. */ + let turnSeq = 0; + /** Tool names by call id, so the end payload can name the tool pi started. */ + const toolNames = new Map(); + /** In-flight observability posts, drained at shutdown so none are lost. */ + const inFlight = new Set>(); + + /** + * Resolve configuration lazily. + * + * pi's extension docs are explicit that a factory may run in invocations that + * never start a session, so factories must not open resources. Reading the + * environment is deferred to the first hook instead. + */ + const ensureConfig = (ctx: ExtensionContext): GatewayConfig => { + config ??= configFromEnv(safeSessionId(ctx)); + return config; + }; + + /** Fire an observability-only hook without charging pi's critical path. */ + const emit = (ctx: ExtensionContext, payload: Record): void => { + const pending = postAndForget(ensureConfig(ctx), payload); + inFlight.add(pending); + void pending.finally(() => inFlight.delete(pending)); + }; + + // --------------------------------------------------------------------------- + // Session lifecycle + // + // pi's session_start/session_shutdown are the session boundary, NOT + // agent_start/agent_end -- one pi session re-enters the agent run many times, + // so treating agent_start as a session start would open a session per retry. + // --------------------------------------------------------------------------- + + pi.on('session_start', async (event: SessionStartEvent, ctx: ExtensionContext) => { + emit(ctx, { hook_event_name: 'session_start', reason: event.reason, cwd: ctx.cwd }); + }); + + pi.on('session_shutdown', async (_event: SessionShutdownEvent, ctx: ExtensionContext) => { + emit(ctx, { hook_event_name: 'session_shutdown' }); + // Drain before the process exits, or trailing spans are lost. + await Promise.allSettled([...inFlight]); + }); + + // --------------------------------------------------------------------------- + // Agent-run lifecycle + // --------------------------------------------------------------------------- + + pi.on('agent_start', async (_event: AgentStartEvent, ctx: ExtensionContext) => { + emit(ctx, { hook_event_name: 'agent_start', attempt_index: attemptIndex }); + attemptIndex += 1; + }); + + pi.on('agent_end', async (event: AgentEndEvent, ctx: ExtensionContext) => { + emit(ctx, { + hook_event_name: 'agent_end', + // Deliberately not a run boundary: pi may re-enter after this. + attempt_index: Math.max(0, attemptIndex - 1), + message_count: Array.isArray(event.messages) ? event.messages.length : 0, + }); + }); + + pi.on('agent_settled', async (_event: AgentSettledEvent, ctx: ExtensionContext) => { + emit(ctx, { hook_event_name: 'agent_settled', attempts: attemptIndex }); + attemptIndex = 0; + }); + + pi.on('turn_start', async (event: TurnStartEvent, ctx: ExtensionContext) => { + emit(ctx, { + hook_event_name: 'turn_start', + // pi's turn_index resets to 0 on re-entry; turn_seq does not, so a + // consumer can still order turns across the whole session. + turn_index: event.turnIndex, + turn_seq: turnSeq, + attempt_index: Math.max(0, attemptIndex - 1), + }); + turnSeq += 1; + }); + + pi.on('turn_end', async (event: TurnEndEvent, ctx: ExtensionContext) => { + emit(ctx, { hook_event_name: 'turn_end', turn_index: event.turnIndex }); + }); + + // --------------------------------------------------------------------------- + // Tool lifecycle + // --------------------------------------------------------------------------- + + /** + * Fires before validation and before `tool_call`, and also for calls that + * never execute. Recorded only so `tool_execution_end` can name its tool; it + * is deliberately not forwarded as a tool start, because doing so would open + * gateway spans for calls pi then discards. + */ + pi.on('tool_execution_start', async (event: ToolExecutionStartEvent, _ctx: ExtensionContext) => { + toolNames.set(event.toolCallId, event.toolName); + }); + + /** + * The governance seam, and the only hook that blocks. + * + * Trap: `emitToolCall` returns on the first `{block: true}`, so an + * earlier-loading extension can block before this handler runs. The call is + * still blocked, but nothing is evaluated and the gateway never sees it. + */ + pi.on( + 'tool_call', + async ( + event: ToolCallEvent, + ctx: ExtensionContext, + ): Promise => { + const active = ensureConfig(ctx); + const outcome = await postHook(active, { + hook_event_name: 'tool_call', + tool_call_id: event.toolCallId, + tool_name: event.toolName, + input: event.input, + }); + + const decision = + outcome.kind === 'fault' ? resolveFault(active, outcome.detail, event.toolName) : outcome; + + // `undefined` is the only correct allow value: a truthy result without + // `block` is inert but overwrites earlier handlers' results. + if (decision.kind !== 'block') return undefined; + return { block: true, reason: decision.reason }; + }, + ); + + /** + * The tool end boundary for every outcome. + * + * `tool_result` does not fire for blocked calls -- they take pi's + * `kind: "immediate"` path and never reach `afterToolCall` -- but + * `tool_execution_end` always fires, with `isError: true`. So this is the only + * hook that closes both allowed and blocked calls. + */ + pi.on('tool_execution_end', async (event: ToolExecutionEndEvent, ctx: ExtensionContext) => { + const toolName = event.toolName || toolNames.get(event.toolCallId) || 'unknown'; + toolNames.delete(event.toolCallId); + emit(ctx, { + hook_event_name: 'tool_execution_end', + tool_call_id: event.toolCallId, + tool_name: toolName, + result: summarize(event.result, event.isError), + status: event.isError ? 'error' : 'ok', + }); + }); +} + +/** pi's session id, with a fallback so a missing manager cannot break loading. */ +function safeSessionId(ctx: ExtensionContext): string { + try { + return ctx.sessionManager?.getSessionId?.() ?? 'unknown-session'; + } catch { + return 'unknown-session'; + } +} + +const MAX_CONTENT_CHARS = 2000; + +/** Keep forwarded tool results small and JSON-safe. */ +function summarize(result: unknown, isError: boolean): unknown { + if (result === null || result === undefined) { + return { content: isError ? 'Tool failed with no result.' : 'Tool completed with no result.' }; + } + if (typeof result === 'string') return { content: truncate(result) }; + if (typeof result === 'object') { + const record = result as Record; + const content = record.content ?? record.output ?? record.text; + return { + content: + typeof content === 'string' + ? truncate(content) + : `Tool ${isError ? 'failed' : 'completed'}.`, + result_keys: Object.keys(record).slice(0, 20), + }; + } + return { content: truncate(String(result)) }; +} + +function truncate(value: string): string { + return value.length <= MAX_CONTENT_CHARS + ? value + : `${value.slice(0, MAX_CONTENT_CHARS)}... [truncated ${value.length - MAX_CONTENT_CHARS} chars]`; +} diff --git a/integrations/pi/package.json b/integrations/pi/package.json new file mode 100644 index 000000000..192fdd3c4 --- /dev/null +++ b/integrations/pi/package.json @@ -0,0 +1,17 @@ +{ + "name": "nemo-relay-pi", + "version": "0.8.0", + "private": true, + "description": "NeMo Relay proof-of-concept extension for the pi coding agent.", + "type": "module", + "repository": { + "type": "git", + "url": "https://github.com/NVIDIA/NeMo-Relay", + "directory": "integrations/pi" + }, + "main": "./index.ts", + "scripts": { + "typecheck": "tsc -p tsconfig.json", + "test": "node --test test/*.test.mjs" + } +} diff --git a/integrations/pi/src/gateway-client.ts b/integrations/pi/src/gateway-client.ts new file mode 100644 index 000000000..7e9036ae0 --- /dev/null +++ b/integrations/pi/src/gateway-client.ts @@ -0,0 +1,154 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * HTTP client for the NeMo Relay CLI gateway's `/hooks/pi` endpoint. + * + * The wire contract, verified against `crates/cli`: + * + * - **Allow** is any 2xx. The adapter returns `{}`; the body is not meaningful. + * - **Block** is `403` with + * `{"error": {"type": "nemo_relay_guardrail_rejected", "reason": ""}}`. + * The rejection comes from the tool conditional-execution guardrail chain that + * the gateway runs in `start_tool`, and `error.reason` is the guardrail's own + * words. pi passes that string to the model verbatim as an error tool result, + * so it must be forwarded unchanged. + * - Anything else is a transport or gateway fault, and is resolved by the + * configured failure policy rather than being reported as a policy decision. + * + * pi awaits extension handlers on its critical path, so every call here is on + * the critical path of the tool it gates. Observability-only hooks are therefore + * sent without awaiting, and only the gating hook blocks. + */ + +/** Outcome of posting one hook to the gateway. */ +export type HookOutcome = + | { kind: 'allow' } + | { kind: 'block'; reason: string } + | { kind: 'fault'; detail: string }; + +export type GatewayConfig = { + /** Base URL of the gateway, e.g. `http://127.0.0.1:4040`. */ + url: string; + /** Per-request timeout in milliseconds. */ + timeoutMs: number; + /** + * What to do when the gateway cannot be reached or errors. + * + * `open` lets the tool run; `closed` blocks it. Defaults to `open` because a + * dead sidecar should not brick the user's agent, matching how the shipped + * `hooks.json` files use `--fail-open` for everything except the pre-tool and + * permission events. + */ + onFault: 'open' | 'closed'; + /** Session identifier sent with every payload. */ + sessionId: string; +}; + +const GUARDRAIL_REJECTION_TYPE = 'nemo_relay_guardrail_rejected'; + +/** Build the routing-identity headers the gateway expects. */ +function headers(config: GatewayConfig): Record { + return { + 'content-type': 'application/json', + // The gateway strips inbound routing-identity headers as an anti-spoofing + // measure and re-derives its own, so this is the only session signal that + // survives -- it must also appear in the payload. + 'x-nemo-relay-session-id': config.sessionId, + }; +} + +/** + * Post one hook and wait for the verdict. + * + * Used only for the gating hook (`tool_call`). Everything else should use + * {@link postAndForget} so pi's critical path is not charged an extra round + * trip for an observability-only event. + */ +export async function postHook( + config: GatewayConfig, + payload: Record, +): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), config.timeoutMs); + try { + const response = await fetch(`${config.url}/hooks/pi`, { + method: 'POST', + headers: headers(config), + body: JSON.stringify({ session_id: config.sessionId, ...payload }), + signal: controller.signal, + }); + + if (response.ok) return { kind: 'allow' }; + + if (response.status === 403) { + const body = await safeJson(response); + const error = body?.error; + if (error?.type === GUARDRAIL_REJECTION_TYPE && typeof error.reason === 'string') { + return { kind: 'block', reason: error.reason }; + } + // A 403 without the guardrail marker is an authorization fault, not a + // policy decision; do not present it to the model as one. + return { kind: 'fault', detail: `gateway returned 403 without a guardrail reason` }; + } + + return { kind: 'fault', detail: `gateway returned HTTP ${response.status}` }; + } catch (error) { + const detail = + error instanceof Error && error.name === 'AbortError' + ? `gateway did not respond within ${config.timeoutMs}ms` + : `gateway request failed: ${error instanceof Error ? error.message : String(error)}`; + return { kind: 'fault', detail }; + } finally { + clearTimeout(timer); + } +} + +/** + * Post one hook without waiting for it. + * + * Returns a promise that never rejects, so a failed observability post cannot + * surface as an unhandled rejection inside pi's TUI. Callers should collect + * these and await them at session shutdown so nothing is lost on exit. + */ +export function postAndForget( + config: GatewayConfig, + payload: Record, +): Promise { + return postHook(config, payload).then( + () => undefined, + () => undefined, + ); +} + +/** Resolve a fault into an allow/block decision using the configured policy. */ +export function resolveFault(config: GatewayConfig, detail: string, toolName: string): HookOutcome { + if (config.onFault === 'open') return { kind: 'allow' }; + return { + kind: 'block', + reason: + `The NeMo Relay policy gateway could not be reached to authorize this ${toolName} call, ` + + `so it was blocked rather than allowed through unchecked. This is an infrastructure fault, ` + + `not a judgement about the request. Details: ${detail}`, + }; +} + +async function safeJson(response: Response): Promise<{ error?: Record } | null> { + try { + return (await response.json()) as { error?: Record }; + } catch { + return null; + } +} + +/** Read gateway configuration from the environment the CLI's launcher sets. */ +export function configFromEnv(sessionId: string): GatewayConfig { + const url = (process.env.NEMO_RELAY_PI_GATEWAY_URL ?? 'http://127.0.0.1:4040').replace(/\/+$/, ''); + const timeoutRaw = Number(process.env.NEMO_RELAY_PI_TIMEOUT_MS); + return { + url, + timeoutMs: Number.isFinite(timeoutRaw) && timeoutRaw > 0 ? timeoutRaw : 5000, + onFault: process.env.NEMO_RELAY_PI_FAIL === 'closed' ? 'closed' : 'open', + sessionId, + }; +} diff --git a/integrations/pi/src/pi-hook-types.ts b/integrations/pi/src/pi-hook-types.ts new file mode 100644 index 000000000..9add97447 --- /dev/null +++ b/integrations/pi/src/pi-hook-types.ts @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Structural mirror of the subset of pi's extension API this integration uses. + * + * Mirrored from pi `v0.84.0` (`a5f43bf8a`), + * `packages/coding-agent/src/core/extensions/types.ts`. Declaring the shapes + * locally -- the same approach `integrations/openclaw/src/openclaw-hook-types.ts` + * takes for its host agent -- keeps this directory buildable without depending + * on the pi package, which matters because pi ships breaking changes through + * *minor* releases and has no major-release channel. + * + * Re-verify these signatures against the pinned pi version before relying on + * them; a silent shape change would show up as missing spans, not a type error. + */ + +/** Fired when an agent loop starts. Carries no run identifier. */ +export type AgentStartEvent = { type: 'agent_start' }; + +/** + * Fired when an agent loop ends. + * + * Note the absence of `willRetry`: the *public session* `agent_end` carries it, + * the extension-facing one does not, and `auto_retry_start`/`_end` never reach + * extensions at all. Detecting a retry here is therefore impossible; close the + * logical run on `agent_settled` instead. + */ +export type AgentEndEvent = { type: 'agent_end'; messages: unknown[] }; + +/** Fired once per logical agent run, from a `finally`. */ +export type AgentSettledEvent = { type: 'agent_settled' }; + +/** Fired at the start of each turn. `turnIndex` resets to 0 on run re-entry. */ +export type TurnStartEvent = { type: 'turn_start'; turnIndex: number; timestamp: number }; + +export type TurnEndEvent = { + type: 'turn_end'; + turnIndex: number; + message: unknown; + toolResults: unknown[]; +}; + +export type SessionStartEvent = { + type: 'session_start'; + reason: 'startup' | 'reload' | 'new' | 'resume' | 'fork'; + previousSessionFile?: string; +}; + +export type SessionShutdownEvent = { type: 'session_shutdown' }; + +/** + * Fired when a tool starts executing. + * + * Fires *before* argument validation and before the `tool_call` hook, and also + * for calls that never execute, so a handle map keyed on this must tolerate a + * miss. `args` are the pre-clone originals. + */ +export type ToolExecutionStartEvent = { + type: 'tool_execution_start'; + toolCallId: string; + toolName: string; + args: unknown; +}; + +export type ToolExecutionEndEvent = { + type: 'tool_execution_end'; + toolCallId: string; + toolName: string; + result: unknown; + isError: boolean; +}; + +/** + * Fired before a tool executes; the only pi hook that can block. + * + * `input` is mutable -- mutating it in place patches the arguments, later + * `tool_call` handlers see earlier mutations, and no re-validation happens + * afterwards. + */ +export type ToolCallEvent = { + type: 'tool_call'; + toolCallId: string; + toolName: string; + input: Record; +}; + +/** Returning `{block: true}` short-circuits the remaining `tool_call` handlers. */ +export type ToolCallEventResult = { + block?: boolean; + reason?: string; +}; + +/** Minimal view of pi's extension context. */ +export type ExtensionContext = { + cwd: string; + mode: string; + hasUI: boolean; + sessionManager: { getSessionId(): string }; +}; + +export type ExtensionHandler = ( + event: TEvent, + ctx: ExtensionContext, +) => TResult | undefined | Promise; + +/** Minimal view of pi's `ExtensionAPI`, limited to what this extension registers. */ +export type ExtensionAPI = { + on(event: 'session_start', handler: ExtensionHandler): void; + on(event: 'session_shutdown', handler: ExtensionHandler): void; + on(event: 'agent_start', handler: ExtensionHandler): void; + on(event: 'agent_end', handler: ExtensionHandler): void; + on(event: 'agent_settled', handler: ExtensionHandler): void; + on(event: 'turn_start', handler: ExtensionHandler): void; + on(event: 'turn_end', handler: ExtensionHandler): void; + on(event: 'tool_execution_start', handler: ExtensionHandler): void; + on(event: 'tool_execution_end', handler: ExtensionHandler): void; + on(event: 'tool_call', handler: ExtensionHandler): void; +}; diff --git a/integrations/pi/test/fixtures/reentry-driver.ts b/integrations/pi/test/fixtures/reentry-driver.ts new file mode 100644 index 000000000..6a38bdd17 --- /dev/null +++ b/integrations/pi/test/fixtures/reentry-driver.ts @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Test fixture: forces exactly one agent-run re-entry. + * + * pi re-enters the agent run from `_handlePostAgentRun` on three paths -- + * provider retry, compaction, and a queued follow-up. The queued-follow-up path + * is the only one an extension can trigger deterministically, and pi documents + * it explicitly: messages queued by an `agent_end` handler "need a + * continuation", so `agent.continue()` runs and a fresh `agent_start` fires with + * `turnIndex` reset to 0. + * + * This drives the *real* re-entry path rather than simulating it, which is the + * point -- the colliding turn indices it produces are pi's, not the fixture's. + * + * Load it alongside the Relay extension: + * pi -e -e + */ +import type { ExtensionAPI } from '../../src/pi-hook-types.ts'; + +/** pi's `sendUserMessage`, which the mirrored type subset does not declare. */ +type ReentryCapableApi = ExtensionAPI & { + sendUserMessage(text: string, options?: { deliverAs?: 'steer' | 'followUp' }): void; +}; + +export default function reentryDriver(pi: ReentryCapableApi): void { + let fired = false; + + pi.on('agent_end', async () => { + // Fire once only: queueing on every agent_end would re-enter forever. + if (fired) return; + fired = true; + pi.sendUserMessage('Now reply with exactly the word: done', { deliverAs: 'followUp' }); + }); +} diff --git a/integrations/pi/test/gateway-client.test.mjs b/integrations/pi/test/gateway-client.test.mjs new file mode 100644 index 000000000..2a80d1399 --- /dev/null +++ b/integrations/pi/test/gateway-client.test.mjs @@ -0,0 +1,205 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Tests the extension's half of the `/hooks/pi` wire contract. + * + * The gateway's half is pinned in Rust by + * `pi_tool_call_hook_rejects_when_conditional_guardrail_blocks` and + * `pi_tool_call_hook_allows_when_no_guardrail_objects` + * (`crates/cli/tests/coverage/shared/server_tests.rs`). These tests run against + * a local HTTP server that reproduces the exact shapes those tests assert, so + * the two halves are checked against the same contract from both sides. + * + * Run: node --test integrations/pi/test/*.test.mjs + */ +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import { after, before, describe, it } from 'node:test'; + +const { postHook, resolveFault, configFromEnv } = await import('../src/gateway-client.ts'); + +/** Start a server that replies with `handler(requestBody)`. */ +function serve(handler) { + const received = []; + const server = createServer((req, res) => { + let body = ''; + req.on('data', (chunk) => { + body += chunk; + }); + req.on('end', () => { + received.push({ url: req.url, headers: req.headers, body: JSON.parse(body || '{}') }); + const { status, payload, delayMs } = handler(received.at(-1)); + const send = () => { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(JSON.stringify(payload ?? {})); + }; + if (delayMs) setTimeout(send, delayMs); + else send(); + }); + }); + return { server, received }; +} + +async function listen(server) { + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + return `http://127.0.0.1:${server.address().port}`; +} + +const baseConfig = (url, overrides = {}) => ({ + url, + timeoutMs: 2000, + onFault: 'open', + sessionId: 'test-session', + ...overrides, +}); + +describe('gateway client wire contract', () => { + let ctx; + let url; + + before(async () => { + ctx = serve((request) => { + const name = request.body.hook_event_name; + if (name === 'slow') return { status: 200, payload: {}, delayMs: 500 }; + if (name === 'boom') return { status: 500, payload: { error: { message: 'kaboom' } } }; + if (name === 'naked-403') return { status: 403, payload: { error: { message: 'nope' } } }; + if (request.body.tool_name === 'read' && request.body.input?.path?.endsWith('.env')) { + // Byte-for-byte the shape CliError::into_response produces. + return { + status: 403, + payload: { + error: { + message: 'guardrail rejected: read .env is blocked; use .env.example', + type: 'nemo_relay_guardrail_rejected', + reason: 'read .env is blocked; use .env.example', + }, + }, + }; + } + return { status: 200, payload: {} }; + }); + url = await listen(ctx.server); + }); + + after(() => ctx.server.close()); + + it('treats 2xx as allow', async () => { + const outcome = await postHook(baseConfig(url), { + hook_event_name: 'tool_call', + tool_name: 'read', + input: { path: 'README.md' }, + }); + assert.deepEqual(outcome, { kind: 'allow' }); + }); + + it('turns a guardrail 403 into a block carrying the reason verbatim', async () => { + const outcome = await postHook(baseConfig(url), { + hook_event_name: 'tool_call', + tool_name: 'read', + input: { path: '/work/.env' }, + }); + assert.equal(outcome.kind, 'block'); + // Verbatim matters: pi passes this straight to the model, so any added + // framing would become part of what the model reads. + assert.equal(outcome.reason, 'read .env is blocked; use .env.example'); + assert.ok(!outcome.reason.startsWith('guardrail rejected:'), 'runtime framing must be stripped'); + }); + + it('does not present a 403 without the guardrail marker as a policy decision', async () => { + // An authorization failure is not a judgement about the request; reporting + // it as one would tell the model a policy considered and refused its call. + const outcome = await postHook(baseConfig(url), { hook_event_name: 'naked-403' }); + assert.equal(outcome.kind, 'fault'); + }); + + it('reports a non-403 error status as a fault, not a block', async () => { + const outcome = await postHook(baseConfig(url), { hook_event_name: 'boom' }); + assert.equal(outcome.kind, 'fault'); + assert.match(outcome.detail, /HTTP 500/); + }); + + it('times out rather than hanging pi\'s critical path', async () => { + const outcome = await postHook(baseConfig(url, { timeoutMs: 50 }), { + hook_event_name: 'slow', + }); + assert.equal(outcome.kind, 'fault'); + assert.match(outcome.detail, /did not respond within 50ms/); + }); + + it('reports an unreachable gateway as a fault', async () => { + // Port 1 is reserved and never listening. + const outcome = await postHook(baseConfig('http://127.0.0.1:1'), { + hook_event_name: 'tool_call', + }); + assert.equal(outcome.kind, 'fault'); + }); + + it('sends the session id in both the header and the payload', async () => { + await postHook(baseConfig(url, { sessionId: 'sess-42' }), { hook_event_name: 'session_start' }); + const last = ctx.received.at(-1); + assert.equal(last.url, '/hooks/pi'); + assert.equal(last.headers['x-nemo-relay-session-id'], 'sess-42'); + // The gateway strips inbound routing-identity headers, so the payload copy + // is what actually survives. + assert.equal(last.body.session_id, 'sess-42'); + }); +}); + +describe('failure policy', () => { + it('fails open by default so a dead sidecar does not brick the agent', () => { + const outcome = resolveFault( + { url: '', timeoutMs: 1, onFault: 'open', sessionId: 's' }, + 'connection refused', + 'read', + ); + assert.deepEqual(outcome, { kind: 'allow' }); + }); + + it('fails closed on request, and says the block is infrastructure not policy', () => { + const outcome = resolveFault( + { url: '', timeoutMs: 1, onFault: 'closed', sessionId: 's' }, + 'connection refused', + 'read', + ); + assert.equal(outcome.kind, 'block'); + assert.match(outcome.reason, /infrastructure fault, not a judgement/); + assert.match(outcome.reason, /connection refused/); + }); +}); + +describe('configFromEnv', () => { + const saved = { ...process.env }; + after(() => { + process.env = saved; + }); + + it('defaults to the gateway default bind and fails open', () => { + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + delete process.env.NEMO_RELAY_PI_TIMEOUT_MS; + delete process.env.NEMO_RELAY_PI_FAIL; + const config = configFromEnv('s1'); + assert.equal(config.url, 'http://127.0.0.1:4040'); + assert.equal(config.timeoutMs, 5000); + assert.equal(config.onFault, 'open'); + }); + + it('strips a trailing slash so the path join cannot double up', () => { + process.env.NEMO_RELAY_PI_GATEWAY_URL = 'http://127.0.0.1:9999/'; + assert.equal(configFromEnv('s1').url, 'http://127.0.0.1:9999'); + }); + + it('ignores a non-numeric or non-positive timeout', () => { + process.env.NEMO_RELAY_PI_TIMEOUT_MS = 'soon'; + assert.equal(configFromEnv('s1').timeoutMs, 5000); + process.env.NEMO_RELAY_PI_TIMEOUT_MS = '0'; + assert.equal(configFromEnv('s1').timeoutMs, 5000); + }); + + it('opts into fail-closed only on the exact value', () => { + process.env.NEMO_RELAY_PI_FAIL = 'closed'; + assert.equal(configFromEnv('s1').onFault, 'closed'); + process.env.NEMO_RELAY_PI_FAIL = 'CLOSED'; + assert.equal(configFromEnv('s1').onFault, 'open'); + }); +}); diff --git a/integrations/pi/tsconfig.json b/integrations/pi/tsconfig.json new file mode 100644 index 000000000..78c8ba64a --- /dev/null +++ b/integrations/pi/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2023"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "types": ["node"], + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitOverride": true, + "skipLibCheck": true, + "noEmit": true, + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true + }, + "include": ["index.ts", "src/**/*.ts", "test/**/*.ts"] +} From e17d997eaf9c50d326ff273ca4f6c99df7321be1 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 12:33:03 -0700 Subject: [PATCH 02/41] fix(cli): map pi turn boundaries and serialize extension hook posts Three fixes found by running pi against a gateway with the ATOF exporter enabled and reading the emitted trace. 1. pi's turn_start/turn_end produced marks, not turn scopes. TurnEnded was only emitted for the hardcoded name "stop", which is Codex and Claude Code vocabulary that pi never sends. The gateway therefore opened one implicit turn covering the whole run and pi's own turn boundaries were lost. ClassificationRules gains a turn_end list. Codex and Claude Code declare &["Stop", "stop"], which preserves their behaviour exactly; pi declares its native turn_end. agent_settled is deliberately not in pi's list -- it marks the end of a logical agent run, which can span several turns, so closing the turn there would merge every re-entry attempt into one. 2. Unawaited hook posts raced, reordering the lifecycle. Firing observability posts concurrently let them arrive out of order. An observed trace had agent_start landing after turn_start and agent_end after agent_settled, and a session_shutdown that overtook an in-flight post closed the session and let the straggler open a second one. The extension now serializes every post through a chain. Observability hooks are enqueued rather than awaited, so pi's critical path is still not charged. The gating hook does await, which also makes it wait for anything queued ahead of it -- worth the latency, because a tool span opened under the wrong turn is simply wrong. 3. A generic arrow function stopped the extension loading. `(job) => ...` in a .ts file is ambiguous with JSX, and pi's jiti loader resolves it that way. pi collects extension load errors rather than aborting, so the extension silently did not run. Declared as a function instead. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- crates/cli/src/agents/claude/adapter.rs | 1 + crates/cli/src/agents/codex/adapter.rs | 1 + crates/cli/src/agents/pi/adapter.rs | 6 +++ crates/cli/src/agents/shared/adapters.rs | 21 ++++++-- integrations/pi/index.ts | 62 ++++++++++++++++++------ 5 files changed, 73 insertions(+), 18 deletions(-) diff --git a/crates/cli/src/agents/claude/adapter.rs b/crates/cli/src/agents/claude/adapter.rs index ed1cdde98..01f477328 100644 --- a/crates/cli/src/agents/claude/adapter.rs +++ b/crates/cli/src/agents/claude/adapter.rs @@ -38,6 +38,7 @@ pub(crate) fn adapt(payload: Value, headers: &HeaderMap) -> AdapterOutcome { "PermissionDenied", "permissionDenied", ], + turn_end: &["Stop", "stop"], }, ); // Response shape is decided by the primary event (first in the vec); secondary events like diff --git a/crates/cli/src/agents/codex/adapter.rs b/crates/cli/src/agents/codex/adapter.rs index 536f778ba..4f7eb77e9 100644 --- a/crates/cli/src/agents/codex/adapter.rs +++ b/crates/cli/src/agents/codex/adapter.rs @@ -27,6 +27,7 @@ pub(crate) fn adapt(payload: Value, headers: &HeaderMap) -> AdapterOutcome { subagent_end: &["subagentStop", "subagentEnd", "subagent_stop"], tool_start: &["preToolUse", "toolStarted", "tool_start"], tool_end: &["postToolUse", "toolEnded", "tool_end", "toolFailed"], + turn_end: &["Stop", "stop"], }, ); AdapterOutcome { diff --git a/crates/cli/src/agents/pi/adapter.rs b/crates/cli/src/agents/pi/adapter.rs index ccaab7f14..0179f962b 100644 --- a/crates/cli/src/agents/pi/adapter.rs +++ b/crates/cli/src/agents/pi/adapter.rs @@ -44,6 +44,12 @@ pub(crate) fn adapt(payload: Value, headers: &HeaderMap) -> AdapterOutcome { subagent_end: &[], tool_start: &["tool_call", "toolCall"], tool_end: &["tool_execution_end", "toolExecutionEnd"], + // pi has an explicit turn boundary, unlike Codex and Claude Code + // which only signal it through `Stop`. `agent_settled` is + // deliberately not here: it marks the end of a logical agent run, + // which can span several turns, and closing the turn there would + // merge every re-entry attempt into one. + turn_end: &["turn_end", "turnEnd"], }, ); AdapterOutcome { diff --git a/crates/cli/src/agents/shared/adapters.rs b/crates/cli/src/agents/shared/adapters.rs index 30e970a3c..bfe8f3b4f 100644 --- a/crates/cli/src/agents/shared/adapters.rs +++ b/crates/cli/src/agents/shared/adapters.rs @@ -40,6 +40,13 @@ pub(super) struct ClassificationRules<'a> { subagent_end: &'a [&'a str], tool_start: &'a [&'a str], tool_end: &'a [&'a str], + /// Hook names that additionally close the turn scope. + /// + /// Claude Code and Codex both spell this `Stop`; pi has an explicit + /// `turn_end`. The event is emitted alongside the primary one so the + /// session manager can close the turn and snapshot ATIF without closing + /// the agent scope. + turn_end: &'a [&'a str], } #[derive(Debug, Clone, PartialEq, Eq, Default)] @@ -707,9 +714,10 @@ fn event_detail_result(payload: &Value, normalized_event: &str) -> Option /// Classify a raw hook event into one or more normalized events. /// /// Most hook events produce a single normalized event from `classify_primary`. -/// The exception is `Stop` for Claude Code and Codex: it emits both the -/// existing `LlmHint` and a `TurnEnded` so the session manager can snapshot ATIF -/// without closing the agent scope. +/// The exception is a turn-end hook -- `Stop` for Claude Code and Codex, +/// `turn_end` for pi -- which emits both the primary event and a `TurnEnded` so +/// the session manager can close the turn and snapshot ATIF without closing the +/// agent scope. /// /// If the primary event is already terminal, the snapshot is skipped to avoid /// double-writing and accidentally recreating an empty session. @@ -754,7 +762,12 @@ fn classify( return vec![NormalizedEvent::HookMark(event)]; } let primary = classify_primary(payload, headers, extractor, rules, &fallback_session_id); - if normalized == "stop" && !primary.is_terminal() { + if rules + .turn_end + .iter() + .any(|name| normalize_name(name) == normalized) + && !primary.is_terminal() + { return vec![ primary, NormalizedEvent::TurnEnded(common_session_event_with_fallback( diff --git a/integrations/pi/index.ts b/integrations/pi/index.ts index 13e1a3abc..6d28df74a 100644 --- a/integrations/pi/index.ts +++ b/integrations/pi/index.ts @@ -71,8 +71,36 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { let turnSeq = 0; /** Tool names by call id, so the end payload can name the tool pi started. */ const toolNames = new Map(); - /** In-flight observability posts, drained at shutdown so none are lost. */ - const inFlight = new Set>(); + + /** + * Serializes every post to the gateway, in hook order. + * + * Firing posts concurrently reorders them: the gateway derives session and + * turn boundaries from arrival order, so a late `agent_start` can land after + * a `turn_start`, and a `session_shutdown` that overtakes an in-flight post + * closes the session and lets the straggler open a second one. Both were + * observed in an acceptance trace before this queue existed. + * + * The chain absorbs failures so one bad post cannot stall the rest, and + * observability hooks still do not block pi -- they are enqueued, not + * awaited. The gating hook does await, which means it also waits for + * anything queued ahead of it; that ordering guarantee is worth the latency, + * because a tool span opened under the wrong turn is simply wrong. + */ + let chain: Promise = Promise.resolve(); + + // Declared as a function rather than a generic arrow: `(...) => ...` in a + // .ts file is ambiguous with JSX, and pi's jiti loader resolves it that way + // and fails to load the extension -- silently, because pi collects extension + // load errors rather than aborting. + function enqueue(job: () => Promise): Promise { + const result = chain.then(job, job); + chain = result.then( + () => undefined, + () => undefined, + ); + return result; + } /** * Resolve configuration lazily. @@ -86,11 +114,10 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { return config; }; - /** Fire an observability-only hook without charging pi's critical path. */ + /** Queue an observability-only hook without charging pi's critical path. */ const emit = (ctx: ExtensionContext, payload: Record): void => { - const pending = postAndForget(ensureConfig(ctx), payload); - inFlight.add(pending); - void pending.finally(() => inFlight.delete(pending)); + const active = ensureConfig(ctx); + void enqueue(() => postAndForget(active, payload)); }; // --------------------------------------------------------------------------- @@ -107,8 +134,10 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { pi.on('session_shutdown', async (_event: SessionShutdownEvent, ctx: ExtensionContext) => { emit(ctx, { hook_event_name: 'session_shutdown' }); - // Drain before the process exits, or trailing spans are lost. - await Promise.allSettled([...inFlight]); + // Drain before the process exits, or trailing spans are lost. Because the + // queue is serial, this also guarantees session_shutdown is the last post + // to reach the gateway rather than merely one of the last. + await chain; }); // --------------------------------------------------------------------------- @@ -178,12 +207,17 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { ctx: ExtensionContext, ): Promise => { const active = ensureConfig(ctx); - const outcome = await postHook(active, { - hook_event_name: 'tool_call', - tool_call_id: event.toolCallId, - tool_name: event.toolName, - input: event.input, - }); + // Enqueued rather than posted directly, so any observability hook fired + // earlier in the same turn reaches the gateway first and the tool span + // opens under the right turn. + const outcome = await enqueue(() => + postHook(active, { + hook_event_name: 'tool_call', + tool_call_id: event.toolCallId, + tool_name: event.toolName, + input: event.input, + }), + ); const decision = outcome.kind === 'fault' ? resolveFault(active, outcome.detail, event.toolName) : outcome; From 12d081feb531b9761ff6cd6c7b030ac3c8a3bd4b Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 14:10:11 -0700 Subject: [PATCH 03/41] fix(pi): do not end the gateway session on /reload pi's session_shutdown carries reason: quit | reload | new | resume | fork. The extension ignored it -- the mirrored type did not even declare the field -- and forwarded a session end for every reason. On /reload that is wrong. pi tears down and rebuilds the extension runtime while the session itself continues with the same session id, so ending the gateway session there closes its session scope and the session_start that follows opens a second one. One logical session silently became two disconnected traces. The handling was also asymmetric: session_start's reason was already forwarded. Now: reload drains the queue and returns without ending the session; quit and the three session-replacement reasons end it and forward the reason, plus targetSessionFile when pi supplies one. Known limitation, documented at the handler: attemptIndex and turnSeq live in the factory closure and pi re-runs the factory on reload with moduleCache: false, so they restart at 0 mid-session. turn_seq is therefore monotonic within a runtime rather than strictly within a session. Rebuilding them would mean replaying the session. Adds test/lifecycle.test.mjs, which drives the extension's handlers against a stub gateway. Nothing exercised them before -- the existing suite covers the wire contract in isolation -- so attempt_index and turn_seq were implemented and demonstrated in a live trace but never pinned. Now covered: turn attribution across a re-entry (colliding turn_index, monotonic turn_seq), attempt-counter reset on agent_settled, strict post ordering, session id on every post, and the shutdown-reason matrix. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- integrations/pi/index.ts | 29 +++- integrations/pi/src/pi-hook-types.ts | 15 +- integrations/pi/test/lifecycle.test.mjs | 222 ++++++++++++++++++++++++ 3 files changed, 263 insertions(+), 3 deletions(-) create mode 100644 integrations/pi/test/lifecycle.test.mjs diff --git a/integrations/pi/index.ts b/integrations/pi/index.ts index 6d28df74a..e1230c8c1 100644 --- a/integrations/pi/index.ts +++ b/integrations/pi/index.ts @@ -132,8 +132,33 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { emit(ctx, { hook_event_name: 'session_start', reason: event.reason, cwd: ctx.cwd }); }); - pi.on('session_shutdown', async (_event: SessionShutdownEvent, ctx: ExtensionContext) => { - emit(ctx, { hook_event_name: 'session_shutdown' }); + /** + * Only some shutdown reasons actually end the session. + * + * `/reload` tears down and rebuilds the extension runtime while the session + * itself continues, with the same session id. Forwarding a session end there + * closes the gateway's session scope, and the `session_start` that follows + * opens a second one -- silently splitting one logical session into two + * disconnected traces. `quit` and the three session-replacement reasons + * (`new`, `resume`, `fork`) do end the session and are forwarded. + * + * Known limitation: `attemptIndex` and `turnSeq` live in this factory's + * closure, and pi re-runs the factory on reload with `moduleCache: false`, so + * they restart at 0 mid-session. Rebuilding them would mean replaying the + * session, which is out of scope here; `turn_seq` is therefore monotonic + * within a runtime, not strictly within a session. + */ + pi.on('session_shutdown', async (event: SessionShutdownEvent, ctx: ExtensionContext) => { + if (event.reason === 'reload') { + // Still drain: posts already queued belong to the continuing session. + await chain; + return; + } + emit(ctx, { + hook_event_name: 'session_shutdown', + reason: event.reason, + ...(event.targetSessionFile ? { target_session_file: event.targetSessionFile } : {}), + }); // Drain before the process exits, or trailing spans are lost. Because the // queue is serial, this also guarantees session_shutdown is the last post // to reach the gateway rather than merely one of the last. diff --git a/integrations/pi/src/pi-hook-types.ts b/integrations/pi/src/pi-hook-types.ts index 9add97447..4e578852f 100644 --- a/integrations/pi/src/pi-hook-types.ts +++ b/integrations/pi/src/pi-hook-types.ts @@ -47,7 +47,20 @@ export type SessionStartEvent = { previousSessionFile?: string; }; -export type SessionShutdownEvent = { type: 'session_shutdown' }; +/** + * Fired when the current session is torn down. + * + * `reason` matters: only `quit` and the session-replacement reasons mean the + * session is actually over. `reload` tears down and rebuilds the extension + * runtime while the session itself continues, so treating it as an end splits + * one logical session into two traces. + */ +export type SessionShutdownEvent = { + type: 'session_shutdown'; + reason: 'quit' | 'reload' | 'new' | 'resume' | 'fork'; + /** Destination session file when shutting down due to session replacement. */ + targetSessionFile?: string; +}; /** * Fired when a tool starts executing. diff --git a/integrations/pi/test/lifecycle.test.mjs b/integrations/pi/test/lifecycle.test.mjs new file mode 100644 index 000000000..7f642a1c4 --- /dev/null +++ b/integrations/pi/test/lifecycle.test.mjs @@ -0,0 +1,222 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Drives the extension's lifecycle handlers and asserts what reaches the gateway. + * + * `gateway-client.test.mjs` covers the wire contract in isolation; nothing + * exercised the handlers themselves, so the identity fields the gateway cannot + * infer -- `attempt_index` and `turn_seq` -- were implemented and demonstrated + * once in a live trace but never pinned. These tests pin them, plus the + * shutdown-reason behaviour. + * + * Run: node --test integrations/pi/test/*.test.mjs + */ +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import { after, before, beforeEach, describe, it } from 'node:test'; + +const extension = (await import('../index.ts')).default; + +/** Collects every payload posted to /hooks/pi. */ +function stubGateway() { + const posts = []; + const server = createServer((req, res) => { + let body = ''; + req.on('data', (c) => { + body += c; + }); + req.on('end', () => { + posts.push(JSON.parse(body || '{}')); + res.writeHead(200, { 'content-type': 'application/json' }); + res.end('{}'); + }); + }); + return { server, posts }; +} + +/** Registers the extension and returns a driver that fires hooks in order. */ +function load() { + const handlers = new Map(); + const pi = { + on(name, handler) { + if (!handlers.has(name)) handlers.set(name, []); + handlers.get(name).push(handler); + }, + }; + extension(pi); + const ctx = { + cwd: '/work', + mode: 'print', + hasUI: false, + sessionManager: { getSessionId: () => 'sess-under-test' }, + }; + return async (name, event = {}) => { + let result; + for (const handler of handlers.get(name) ?? []) { + result = await handler({ type: name, ...event }, ctx); + } + return result; + }; +} + +const named = (posts, name) => posts.filter((p) => p.hook_event_name === name); + +describe('lifecycle identity the gateway cannot infer', () => { + let ctx; + let url; + + before(async () => { + ctx = stubGateway(); + await new Promise((r) => ctx.server.listen(0, '127.0.0.1', r)); + url = `http://127.0.0.1:${ctx.server.address().port}`; + process.env.NEMO_RELAY_PI_GATEWAY_URL = url; + }); + + after(() => { + ctx.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + }); + + beforeEach(() => { + ctx.posts.length = 0; + }); + + it('attributes each turn to its attempt across an agent-run re-entry', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + // Attempt 0, two turns. + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 1 }); + await fire('turn_end', { turnIndex: 0 }); + await fire('turn_start', { turnIndex: 1, timestamp: 2 }); + await fire('turn_end', { turnIndex: 1 }); + await fire('agent_end', { messages: [] }); + // Re-entry: pi resets turnIndex to 0. + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 3 }); + await fire('turn_end', { turnIndex: 0 }); + await fire('agent_end', { messages: [] }); + await fire('agent_settled'); + await fire('session_shutdown', { reason: 'quit' }); + + const starts = named(ctx.posts, 'turn_start'); + assert.equal(starts.length, 3); + + // pi's turn_index collides across the re-entry... + assert.deepEqual( + starts.map((p) => p.turn_index), + [0, 1, 0], + ); + // ...while turn_seq stays monotonic and attempt_index attributes each turn. + assert.deepEqual( + starts.map((p) => p.turn_seq), + [0, 1, 2], + ); + assert.deepEqual( + starts.map((p) => p.attempt_index), + [0, 0, 1], + ); + + assert.deepEqual( + named(ctx.posts, 'agent_start').map((p) => p.attempt_index), + [0, 1], + ); + assert.equal(named(ctx.posts, 'agent_settled')[0].attempts, 2); + }); + + it('resets the attempt counter on agent_settled so a second prompt starts at 0', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('agent_start'); + await fire('agent_end', { messages: [] }); + await fire('agent_settled'); + await fire('agent_start'); + await fire('session_shutdown', { reason: 'quit' }); + + assert.deepEqual( + named(ctx.posts, 'agent_start').map((p) => p.attempt_index), + [0, 0], + ); + }); + + it('posts every hook in order, never concurrently reordered', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 1 }); + await fire('turn_end', { turnIndex: 0 }); + await fire('agent_end', { messages: [] }); + await fire('agent_settled'); + await fire('session_shutdown', { reason: 'quit' }); + + assert.deepEqual( + ctx.posts.map((p) => p.hook_event_name), + [ + 'session_start', + 'agent_start', + 'turn_start', + 'turn_end', + 'agent_end', + 'agent_settled', + 'session_shutdown', + ], + ); + }); + + it('carries the session id on every post', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('session_shutdown', { reason: 'quit' }); + assert.ok(ctx.posts.length > 0); + for (const post of ctx.posts) { + assert.equal(post.session_id, 'sess-under-test'); + } + }); +}); + +describe('session_shutdown reason', () => { + let ctx; + + before(async () => { + ctx = stubGateway(); + await new Promise((r) => ctx.server.listen(0, '127.0.0.1', r)); + process.env.NEMO_RELAY_PI_GATEWAY_URL = `http://127.0.0.1:${ctx.server.address().port}`; + }); + + after(() => { + ctx.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + }); + + beforeEach(() => { + ctx.posts.length = 0; + }); + + it('does not end the session on /reload, which would split one session into two traces', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('session_shutdown', { reason: 'reload' }); + assert.deepEqual(named(ctx.posts, 'session_shutdown'), []); + // The queue is still drained, so nothing already posted is lost. + assert.equal(named(ctx.posts, 'session_start').length, 1); + }); + + for (const reason of ['quit', 'new', 'resume', 'fork']) { + it(`ends the session on ${reason}, and forwards the reason`, async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('session_shutdown', { reason }); + const ends = named(ctx.posts, 'session_shutdown'); + assert.equal(ends.length, 1); + assert.equal(ends[0].reason, reason); + }); + } + + it('forwards the replacement target when pi supplies one', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('session_shutdown', { reason: 'fork', targetSessionFile: '/s/next.jsonl' }); + assert.equal(named(ctx.posts, 'session_shutdown')[0].target_session_file, '/s/next.jsonl'); + }); +}); From 18c6acfd6a4f64d9212e771b32effbd41b7be702 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 14:14:57 -0700 Subject: [PATCH 04/41] fix(cli): stop claiming pi model traffic is redirected `nemo-relay launch pi` printed a note asserting that model traffic is redirected by the extension registering a gateway-backed provider. It is not: nothing registers a provider yet, so pi's model calls go straight to the provider and the gateway sees no LLM traffic at all. The note now says what is actually true -- tool and turn activity is reported, model calls are not routed, and redirection needs the extension to register a provider because pi has no base-URL flag. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- crates/cli/src/agents/pi/launch.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/cli/src/agents/pi/launch.rs b/crates/cli/src/agents/pi/launch.rs index b7325444d..dde07002f 100644 --- a/crates/cli/src/agents/pi/launch.rs +++ b/crates/cli/src/agents/pi/launch.rs @@ -49,9 +49,14 @@ pub(crate) fn prepare(launch: &mut PreparedAgentLaunch, gateway_url: &str) -> Re ["-e".to_string(), rendered], ); + // Do not claim redirection here: the extension does not yet register a + // gateway-backed provider, so model calls still go straight to the + // provider. Only tool and turn activity reaches Relay today. launch.notes.push( - "pi model traffic is redirected by the NeMo Relay extension registering a gateway-backed \ - provider; pi has no base-URL flag or generic environment override" + "pi tool and turn activity is reported to NeMo Relay by the extension; model calls are \ + NOT yet routed through the gateway, so there are no LLM spans. pi has no base-URL flag \ + or generic environment override, so redirection requires the extension to register a \ + gateway-backed provider" .to_string(), ); Ok(()) From 9430725fa4c26754072cdbcd377d6ed9156decb9 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 14:46:40 -0700 Subject: [PATCH 05/41] docs(pi): correct the turn hook row in the extension README The hook table grouped `turn_start` and `turn_end` into one row and claimed both carry `turn_seq`. Only `turn_start` does, alongside `attempt_index`; `turn_end` posts `turn_index` alone (`integrations/pi/index.ts:191-205`). Split the row so each boundary states what it actually carries. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- integrations/pi/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integrations/pi/README.md b/integrations/pi/README.md index 8ac982194..6f228e905 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -89,7 +89,8 @@ per-call state is keyed by `toolCallId`, the only correlator pi provides. | `session_start` / `session_shutdown` | session boundary | **Not** `agent_start`/`agent_end` — those repeat on re-entry | | `agent_start` / `agent_end` | attempt markers | Carry `attempt_index`; not a run boundary | | `agent_settled` | logical run boundary | Fires exactly once, from a `finally` | -| `turn_start` / `turn_end` | turn boundary | Carries `turn_index` **and** `turn_seq` | +| `turn_start` | turn boundary (open) | Carries `turn_index`, `turn_seq` and `attempt_index` | +| `turn_end` | turn boundary (close) | Carries `turn_index` only | | `tool_call` | tool start, and the gate | The only blocking hook | | `tool_execution_end` | tool end | For **every** outcome, including blocked | | `tool_execution_start` | *not forwarded* | Fires before validation and for calls that never execute | From b7f393a6e80b851ebd77c9920f0aabb0427ce86d Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 15:17:13 -0700 Subject: [PATCH 06/41] feat(pi): open turn scopes at pi's boundary and attribute every hook Closes the two remaining M3 gaps: RELAY-730's turn classification and compaction forwarding, and RELAY-729's attribution. 1. turn_start is classified. Only turn_end was mapped, so the gateway opened the turn implicitly on whichever event arrived first -- agent_start on the first attempt, turn_start on later ones -- and trailing agent_end/agent_settled marks opened an extra empty turn after every run. NormalizedEvent gains TurnStarted and ClassificationRules a turn_start list; Codex and Claude Code declare an empty one and keep their lazily opened turns. Classifying the open is necessary but not sufficient: on its own it adds a *leading* empty turn holding the agent_start mark, because mark() forces a turn open. So for harnesses that report a turn start, a mark arriving between turns is now recorded on the session scope instead. That is what removes the empty turn at both ends, verified by reverting the guard and watching the new test report three turn scopes where pi reported one. 2. Attribution reaches tool spans. tool_call and tool_execution_end carried neither attempt_index nor turn_seq, and turn_end carried turn_index but not turn_seq, so a tool call could be tied to an attempt only by reading arrival order -- which stops working the moment two attempts overlap. The extension now sends both on every attributable hook, and agent_settled sends attempt_index alongside the attempts count. Sending them was not enough. Mark events record the raw payload as their data, but tool spans are built from the extracted call id, name, arguments, result and metadata and drop ToolEvent::payload entirely, so the keys would have been accepted on the wire and silently discarded. PiPayloadExtractor::metadata now promotes the two numeric counters into event metadata. The same promotion puts attribution on the turn scope rather than only on a mark inside it, so "which attempt did this turn belong to" is answerable by walking the scope tree. pi's own turn_index is deliberately not promoted: the gateway assigns its own to the turn scope and the two would collide. 3. Compaction is forwarded. session_before_compact was in none of the three layers. Both halves are now forwarded: session_compact classifies as Compaction, which the runtime treats as proof the context was rebuilt (it marks the owning agent fresh), and session_before_compact stays a mark because it announces an intent any later-loading extension can still cancel. Its willRetry is the only advance notice pi gives an extension that the agent run is about to re-enter. Also drops tool_execution_start from the descriptor's hook_events -- the extension registers it, but only to remember a tool name for the matching end, and never posts it. Verified against a live pi 0.84.0 session with a real model: two turns, both turn_source: turn_start, the read span nested under its turn carrying attempt_index and turn_seq, and the run-level marks on the session scope with no empty trailing turn. Also driven through the hook route with three concurrent tools closing out of submission order and a forced re-entry, where pi's turn_index collides at 0 while turn_seq and attempt_index stay unambiguous. Green: 1163 + 12 + 102 Rust, 29 Node, tsc clean, pre-commit clean apart from cargo-deny/gofmt/go-vet, which are not installed on this machine. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- crates/cli/src/agents/claude/adapter.rs | 5 + crates/cli/src/agents/codex/adapter.rs | 5 + crates/cli/src/agents/pi/adapter.rs | 23 +- crates/cli/src/agents/pi/mod.rs | 7 +- crates/cli/src/agents/shared/adapters.rs | 92 +++++++- crates/cli/src/agents/shared/alignment.rs | 6 + crates/cli/src/events/mod.rs | 26 ++- crates/cli/src/sessions/correlation.rs | 1 + crates/cli/src/sessions/mod.rs | 38 +++- .../tests/coverage/agents/adapters_tests.rs | 201 ++++++++++++++++- .../tests/coverage/agents/alignment_tests.rs | 4 +- .../coverage/agents/coding_agent_tests.rs | 2 +- .../tests/coverage/shared/session_tests.rs | 202 ++++++++++++++++++ integrations/pi/README.md | 69 ++++-- integrations/pi/index.ts | 100 ++++++++- integrations/pi/src/pi-hook-types.ts | 29 +++ integrations/pi/test/lifecycle.test.mjs | 165 ++++++++++++++ 17 files changed, 935 insertions(+), 40 deletions(-) diff --git a/crates/cli/src/agents/claude/adapter.rs b/crates/cli/src/agents/claude/adapter.rs index 01f477328..25753137a 100644 --- a/crates/cli/src/agents/claude/adapter.rs +++ b/crates/cli/src/agents/claude/adapter.rs @@ -38,7 +38,12 @@ pub(crate) fn adapt(payload: Value, headers: &HeaderMap) -> AdapterOutcome { "PermissionDenied", "permissionDenied", ], + // Claude Code reports only the close of a turn (`Stop`); its turns stay + // lazily opened, and `PreCompact`/`PostCompact` are matched by the + // shared fallback rather than by an adapter-specific rule. + turn_start: &[], turn_end: &["Stop", "stop"], + compaction: &[], }, ); // Response shape is decided by the primary event (first in the vec); secondary events like diff --git a/crates/cli/src/agents/codex/adapter.rs b/crates/cli/src/agents/codex/adapter.rs index 4f7eb77e9..cc059c611 100644 --- a/crates/cli/src/agents/codex/adapter.rs +++ b/crates/cli/src/agents/codex/adapter.rs @@ -27,7 +27,12 @@ pub(crate) fn adapt(payload: Value, headers: &HeaderMap) -> AdapterOutcome { subagent_end: &["subagentStop", "subagentEnd", "subagent_stop"], tool_start: &["preToolUse", "toolStarted", "tool_start"], tool_end: &["postToolUse", "toolEnded", "tool_end", "toolFailed"], + // Codex reports only the close of a turn (`Stop`); its turns stay + // lazily opened, and `PreCompact`/`PostCompact` are matched by the + // shared fallback rather than by an adapter-specific rule. + turn_start: &[], turn_end: &["Stop", "stop"], + compaction: &[], }, ); AdapterOutcome { diff --git a/crates/cli/src/agents/pi/adapter.rs b/crates/cli/src/agents/pi/adapter.rs index 0179f962b..e91dbc426 100644 --- a/crates/cli/src/agents/pi/adapter.rs +++ b/crates/cli/src/agents/pi/adapter.rs @@ -24,7 +24,10 @@ use crate::events::AgentKind; /// times (provider retry, compaction, queued follow-up), so treating /// `agent_start` as a session start would open a session per retry. /// - `agent_settled` is the only pi event that fires exactly once per logical -/// agent run, so it is the turn-boundary snapshot rather than `agent_end`. +/// agent run, which makes it the run boundary rather than `agent_end` -- but +/// deliberately *not* a turn boundary: a logical run spans several turns. +/// - pi reports both ends of a turn, so the turn scope opens at `turn_start` +/// instead of being opened implicitly by whichever event arrives first. /// - `tool_call` is the gating hook and maps to tool start. `tool_execution_start` /// is deliberately NOT mapped: it fires before validation and before /// `tool_call`, including for calls that never execute, so using it to open a @@ -44,12 +47,20 @@ pub(crate) fn adapt(payload: Value, headers: &HeaderMap) -> AdapterOutcome { subagent_end: &[], tool_start: &["tool_call", "toolCall"], tool_end: &["tool_execution_end", "toolExecutionEnd"], - // pi has an explicit turn boundary, unlike Codex and Claude Code - // which only signal it through `Stop`. `agent_settled` is - // deliberately not here: it marks the end of a logical agent run, - // which can span several turns, and closing the turn there would - // merge every re-entry attempt into one. + // pi has an explicit turn boundary at both ends, unlike Codex and + // Claude Code which only signal it through `Stop`. Classifying the + // open as well as the close is what stops the gateway inventing a + // turn at whichever event happens to arrive first. + turn_start: &["turn_start", "turnStart"], + // `agent_settled` is deliberately not here: it marks the end of a + // logical agent run, which can span several turns, and closing the + // turn there would merge every re-entry attempt into one. turn_end: &["turn_end", "turnEnd"], + // Only the *completed* compaction. `session_before_compact` stays a + // mark: it announces an intent that any later-loading extension can + // still cancel, and the runtime treats a compaction event as proof + // the context was actually rebuilt. + compaction: &["session_compact", "sessionCompact"], }, ); AdapterOutcome { diff --git a/crates/cli/src/agents/pi/mod.rs b/crates/cli/src/agents/pi/mod.rs index 5e841f0f8..02d5fadf3 100644 --- a/crates/cli/src/agents/pi/mod.rs +++ b/crates/cli/src/agents/pi/mod.rs @@ -28,16 +28,21 @@ pub(super) const DESCRIPTOR: AgentDescriptor = AgentDescriptor { // channel, so this floor is the version the integration was verified // against rather than a lower bound that is expected to keep holding. minimum_version: (0, 84, 0), + // The hooks the extension actually posts to `/hooks/pi`, which is narrower than the set it + // registers with pi. `tool_execution_start` is registered but never forwarded -- it fires + // before validation and for calls that never execute, so it is used only to remember a tool + // name for the matching end -- and listing it here described a hook the gateway never sees. hook_events: &[ "session_start", "session_shutdown", + "session_before_compact", + "session_compact", "agent_start", "agent_end", "agent_settled", "turn_start", "turn_end", "tool_call", - "tool_execution_start", "tool_execution_end", ], }; diff --git a/crates/cli/src/agents/shared/adapters.rs b/crates/cli/src/agents/shared/adapters.rs index bfe8f3b4f..f19643131 100644 --- a/crates/cli/src/agents/shared/adapters.rs +++ b/crates/cli/src/agents/shared/adapters.rs @@ -40,6 +40,12 @@ pub(super) struct ClassificationRules<'a> { subagent_end: &'a [&'a str], tool_start: &'a [&'a str], tool_end: &'a [&'a str], + /// Hook names that open the turn scope at the harness's own boundary. + /// + /// Empty for Claude Code and Codex, which report only `Stop`; their turns + /// stay lazily opened by the first tool, LLM, or mark event of the turn. + /// pi reports `turn_start`, so its turn scope starts where pi says it does. + turn_start: &'a [&'a str], /// Hook names that additionally close the turn scope. /// /// Claude Code and Codex both spell this `Stop`; pi has an explicit @@ -47,6 +53,14 @@ pub(super) struct ClassificationRules<'a> { /// session manager can close the turn and snapshot ATIF without closing /// the agent scope. turn_end: &'a [&'a str], + /// Hook names that report a completed context compaction. + /// + /// Empty for Claude Code and Codex, whose `PreCompact`/`PostCompact` names + /// are already matched by the shared fallback. pi spells the *completed* + /// compaction `session_compact`; its `session_before_compact` deliberately + /// stays a plain mark, because any extension loaded after this one can + /// still cancel the compaction it announces. + compaction: &'a [&'a str], } #[derive(Debug, Clone, PartialEq, Eq, Default)] @@ -199,9 +213,14 @@ pub(super) static CODEX_PAYLOAD_EXTRACTOR: CodexPayloadExtractor = CodexPayloadE pub(super) static PI_PAYLOAD_EXTRACTOR: PiPayloadExtractor = PiPayloadExtractor; /// pi hooks are emitted by a NeMo Relay-authored extension, so the payload uses -/// the canonical key names and needs no path deviations. The one override is the -/// session-header policy: pi is not Claude Code installed mode and must not -/// adopt an `x-claude-code-session-id` that happens to be in the environment. +/// the canonical key names and needs no path deviations. Two overrides: +/// +/// - the session-header policy, because pi is not Claude Code installed mode and +/// must not adopt an `x-claude-code-session-id` that happens to be in the +/// environment; +/// - attempt attribution, because the gateway's session model is flat +/// (session -> turn -> tool/llm) and carries pi's agent-run structure as +/// metadata instead of as scopes. impl AgentPayloadExtractor for PiPayloadExtractor { fn session_header_policy(&self) -> SessionHeaderPolicy { SessionHeaderPolicy::RelayOnly @@ -210,8 +229,51 @@ impl AgentPayloadExtractor for PiPayloadExtractor { fn tool_paths(&self) -> &'static ToolPathSet { PI_TOOL_PATHS } + + /// Promote pi's attempt attribution out of the payload and into metadata. + /// + /// This is load-bearing rather than cosmetic. Mark events carry the raw + /// payload as their `data`, so attribution sent on `agent_start` or + /// `turn_start` is already visible there -- but tool events do not: the + /// session manager builds tool spans from the extracted call id, name, + /// arguments, result, and `metadata`, and drops `ToolEvent::payload` + /// entirely. Without this promotion, `attempt_index` and `turn_seq` on + /// `tool_call` and `tool_execution_end` would be accepted on the wire and + /// then silently discarded. + /// + /// The same promotion is what puts attribution on the *turn scope* rather + /// than only on a mark inside it, so "which attempt did this turn belong + /// to" is answerable by walking the scope tree. + fn metadata( + &self, + payload: &Value, + headers: &HeaderMap, + kind: AgentKind, + event_name: &str, + ) -> Value { + let mut metadata = agent_metadata(payload, headers, kind, event_name); + if let Some(object) = metadata.as_object_mut() { + for key in PI_ATTRIBUTION_KEYS { + // Numbers only: these are counters, and accepting a string here + // would put an untyped field into observability metadata that + // consumers then have to defend against. + if let Some(value) = value_at(payload, &[key]).filter(Value::is_number) { + object.insert((*key).into(), value); + } + } + } + metadata + } } +/// pi attribution counters promoted from hook payloads into event metadata. +/// +/// `attempt_index` counts agent-run re-entries within one prompt; `turn_seq` is +/// session-monotonic where pi's own `turn_index` resets to 0 on every re-entry. +/// pi's `turn_index` is deliberately absent: the gateway assigns its own +/// `turn_index` to the turn scope and promoting pi's would collide with it. +const PI_ATTRIBUTION_KEYS: &[&str] = &["attempt_index", "turn_seq"]; + /// Claude Code reports its native tool identifier as `tool_use_id`, so it uses /// a tool path set that prefers that key. Every other hook field matches the /// canonical defaults (including the installed-mode session-header policy). @@ -901,6 +963,30 @@ fn classify_primary( extractor, fallback_session_id, )) + } else if rules + .turn_start + .iter() + .any(|name| normalize_name(name) == normalized) + { + NormalizedEvent::TurnStarted(common_session_event_with_fallback( + payload, + headers, + rules.kind, + extractor, + fallback_session_id, + )) + } else if rules + .compaction + .iter() + .any(|name| normalize_name(name) == normalized) + { + NormalizedEvent::Compaction(common_session_event_with_fallback( + payload, + headers, + rules.kind, + extractor, + fallback_session_id, + )) } else { match normalized.as_str() { "afteragentresponse" | "agentresponse" | "assistantresponse" | "afteragentthought" diff --git a/crates/cli/src/agents/shared/alignment.rs b/crates/cli/src/agents/shared/alignment.rs index 94e03af54..bb94223f5 100644 --- a/crates/cli/src/agents/shared/alignment.rs +++ b/crates/cli/src/agents/shared/alignment.rs @@ -684,6 +684,12 @@ pub(crate) fn route_event_through_alias( }), Some(child_session_id), ), + // Unlike `TurnEnded`, a turn start is never reused as a subagent-completion signal, so it + // reports no child session id -- there is nothing for the parent to close on it. + NormalizedEvent::TurnStarted(mut event) => { + route_session_event(&mut event, &alias, metadata); + (NormalizedEvent::TurnStarted(event), None) + } NormalizedEvent::TurnEnded(mut event) => { route_session_event(&mut event, &alias, metadata); (NormalizedEvent::TurnEnded(event), Some(child_session_id)) diff --git a/crates/cli/src/events/mod.rs b/crates/cli/src/events/mod.rs index c5743f98a..0535249d6 100644 --- a/crates/cli/src/events/mod.rs +++ b/crates/cli/src/events/mod.rs @@ -24,12 +24,34 @@ impl AgentKind { Self::Gateway => "gateway", } } + + // Whether this harness reports the *opening* of a conversation turn, not just its close. + // + // Codex and Claude Code only signal a turn boundary on `Stop`, so the gateway has to open + // turns lazily -- the first tool call, LLM call, or mark after a close starts the next one. + // pi emits a native `turn_start`, which is classified into `NormalizedEvent::TurnStarted` + // and opens the scope at pi's own boundary. For those harnesses a mark arriving *between* + // turns must not manufacture one: doing so left an empty trailing turn holding nothing but + // the `agent_end`/`agent_settled` marks at the end of every run. + // + // Kept next to `as_str` rather than on the agent descriptor because the session manager + // works in terms of `AgentKind`; the matching hook names live in each adapter's + // `ClassificationRules::turn_start`. + pub(crate) const fn has_explicit_turn_start(self) -> bool { + matches!(self, Self::Pi) + } } #[derive(Debug, Clone, PartialEq)] pub(crate) enum NormalizedEvent { AgentStarted(SessionEvent), AgentEnded(SessionEvent), + /// Conversation-turn boundary that opens the turn scope at the harness's own signal. + /// + /// Only emitted for harnesses that report one (pi's `turn_start`). Without it the gateway + /// opens turns lazily on the first tool, LLM, or mark event, which puts the boundary + /// wherever traffic happens to arrive rather than where the harness says the turn begins. + TurnStarted(SessionEvent), /// Conversation-turn boundary that the gateway uses to snapshot ATIF without closing the /// agent scope. Emitted alongside `LlmHint` for `Stop` hooks (Claude/Codex). /// Required for Codex transparent runs because Codex has no reliable `SessionEnd`-equivalent @@ -60,6 +82,7 @@ impl NormalizedEvent { match self { Self::AgentStarted(event) | Self::AgentEnded(event) + | Self::TurnStarted(event) | Self::TurnEnded(event) | Self::PromptSubmitted(event) | Self::Compaction(event) @@ -72,7 +95,8 @@ impl NormalizedEvent { } pub(crate) fn is_terminal(&self) -> bool { - // TurnEnded is intentionally NOT terminal — the agent scope stays open across turns. + // TurnStarted/TurnEnded are intentionally NOT terminal — the agent scope stays open + // across turns. matches!( self, Self::AgentEnded(_) | Self::SubagentEnded(_) | Self::ToolEnded(_) diff --git a/crates/cli/src/sessions/correlation.rs b/crates/cli/src/sessions/correlation.rs index 495c1a6aa..85d2e41d8 100644 --- a/crates/cli/src/sessions/correlation.rs +++ b/crates/cli/src/sessions/correlation.rs @@ -298,6 +298,7 @@ pub(super) fn event_agent_kind(event: &NormalizedEvent) -> AgentKind { match event { NormalizedEvent::AgentStarted(event) | NormalizedEvent::AgentEnded(event) + | NormalizedEvent::TurnStarted(event) | NormalizedEvent::TurnEnded(event) | NormalizedEvent::PromptSubmitted(event) | NormalizedEvent::Compaction(event) diff --git a/crates/cli/src/sessions/mod.rs b/crates/cli/src/sessions/mod.rs index 1c92e4435..b031c5df2 100644 --- a/crates/cli/src/sessions/mod.rs +++ b/crates/cli/src/sessions/mod.rs @@ -796,6 +796,7 @@ impl Session { match event { NormalizedEvent::AgentStarted(event) => self.start_agent(event).map(|()| None), NormalizedEvent::AgentEnded(event) => self.end_agent(event).await, + NormalizedEvent::TurnStarted(event) => self.start_turn_boundary(event).await, NormalizedEvent::TurnEnded(event) => self.end_turn(event).await, NormalizedEvent::SubagentStarted(event) => { self.start_subagent(event).await.map(|()| None) @@ -1034,6 +1035,27 @@ impl Session { Ok(subscriber_delivery) } + // Opens a turn at the harness's own `turn_start`, for harnesses that report one. + // + // Distinct from `start_turn`, which handles a *prompt* and has to tolerate a prompt arriving + // while a turn is open. A turn-start hook is unambiguous: whatever is still open belongs to + // the previous turn and is closed first, so a dropped or missing `turn_end` degrades into one + // superseded turn rather than merging two turns into one. + async fn start_turn_boundary( + &mut self, + event: SessionEvent, + ) -> Result, CliError> { + let mut subscriber_delivery = None; + if self.turn_scope.is_some() { + let (_, delivery) = self + .close_turn_for_reason("superseded_by_next_turn") + .await?; + subscriber_delivery = delivery; + } + self.open_turn(event.metadata, event.payload, "turn_start")?; + Ok(subscriber_delivery) + } + // Lazily creates an implicit turn when gateway/tool/LLM activity arrives before a prompt hook. // This keeps direct gateway traffic and sparse hook streams bounded by the same lifecycle as // prompt-driven turns. @@ -1630,10 +1652,20 @@ impl Session { .filter(|subagent_id| self.subagents.contains_key(subagent_id)) } - // Emits a mark event after ensuring the turn scope exists. Generic and unknown hooks use this - // path so unsupported agent events remain visible without changing scope structure. + // Emits a mark event after ensuring an enclosing scope exists. Generic and unknown hooks use + // this path so unsupported agent events remain visible without changing scope structure. + // + // Which scope encloses it depends on the harness. Codex and Claude Code report no turn start, + // so a mark has to open the turn it belongs to or it would have nowhere to land. pi does + // report one, and for it a mark arriving between turns is genuinely between turns -- run-level + // events such as `agent_end` and `agent_settled` trail the last `turn_end`, and opening a turn + // for them produced an empty turn scope at the end of every run. fn mark(&mut self, name: &str, event_payload: SessionEvent) -> Result<(), CliError> { - self.ensure_turn_started(event_payload.metadata.clone())?; + if self.agent_kind.has_explicit_turn_start() { + self.ensure_agent_started(event_payload.metadata.clone())?; + } else { + self.ensure_turn_started(event_payload.metadata.clone())?; + } emit_mark_event( EmitMarkEventParams::builder() .name(name) diff --git a/crates/cli/tests/coverage/agents/adapters_tests.rs b/crates/cli/tests/coverage/agents/adapters_tests.rs index 7dc2cc576..0ddc31abc 100644 --- a/crates/cli/tests/coverage/agents/adapters_tests.rs +++ b/crates/cli/tests/coverage/agents/adapters_tests.rs @@ -5,7 +5,7 @@ use axum::http::HeaderMap; use serde_json::json; use super::*; -use crate::agents::shared::adapters::{claude_code, codex}; +use crate::agents::shared::adapters::{claude_code, codex, pi}; #[test] fn maps_claude_canonical_tool_payload() { @@ -1055,3 +1055,202 @@ fn json_path_lookups_handle_empty_strings_arrays_and_deep_nesting() { None ); } + +// -------------------------------------------------------------------------- +// pi turn boundaries, compaction, and attempt attribution +// -------------------------------------------------------------------------- + +// pi is the only harness that reports the *opening* of a turn. Before this was classified the +// gateway opened the turn implicitly on whichever event arrived first, which was `agent_start` +// on the first attempt and `turn_start` on later ones -- the same trace, two different +// boundaries. +#[test] +fn pi_turn_start_opens_the_turn_and_turn_end_closes_it() { + let started = pi::adapt( + json!({ + "session_id": "pi-session", + "hook_event_name": "turn_start", + "turn_index": 0, + "turn_seq": 3, + "attempt_index": 1 + }), + &HeaderMap::new(), + ); + assert!( + matches!(started.events.as_slice(), [NormalizedEvent::TurnStarted(_)]), + "pi turn_start must open the turn scope. events: {:?}", + started.events + ); + + let ended = pi::adapt( + json!({ + "session_id": "pi-session", + "hook_event_name": "turn_end", + "turn_index": 0, + "turn_seq": 3, + "attempt_index": 1 + }), + &HeaderMap::new(), + ); + // The close still emits its primary event plus the shared TurnEnded, exactly as `Stop` does. + assert!( + ended + .events + .iter() + .any(|event| matches!(event, NormalizedEvent::TurnEnded(_))), + "pi turn_end must close the turn scope. events: {:?}", + ended.events + ); +} + +// `agent_settled` ends a logical agent run, which can span several turns. Classifying it as a +// turn boundary would merge every re-entry attempt into one turn. +#[test] +fn pi_agent_lifecycle_hooks_stay_marks_not_turn_boundaries() { + for event_name in ["agent_start", "agent_end", "agent_settled"] { + let outcome = pi::adapt( + json!({ "session_id": "pi-session", "hook_event_name": event_name }), + &HeaderMap::new(), + ); + assert!( + matches!(outcome.events.as_slice(), [NormalizedEvent::HookMark(_)]), + "{event_name} must stay a mark. events: {:?}", + outcome.events + ); + } +} + +// pi announces a compaction before doing it, and any extension loading after this one can still +// cancel it. Only the completed event is a compaction to the runtime, which treats one as proof +// the context was rebuilt. +#[test] +fn pi_compaction_is_classified_only_once_it_has_happened() { + let announced = pi::adapt( + json!({ + "session_id": "pi-session", + "hook_event_name": "session_before_compact", + "reason": "threshold", + "will_retry": false + }), + &HeaderMap::new(), + ); + assert!( + matches!(announced.events.as_slice(), [NormalizedEvent::HookMark(_)]), + "session_before_compact must stay a mark. events: {:?}", + announced.events + ); + + let completed = pi::adapt( + json!({ + "session_id": "pi-session", + "hook_event_name": "session_compact", + "reason": "threshold", + "will_retry": true + }), + &HeaderMap::new(), + ); + assert!( + matches!( + completed.events.as_slice(), + [NormalizedEvent::Compaction(_)] + ), + "session_compact must be a compaction. events: {:?}", + completed.events + ); +} + +// The promotion is what makes attribution survive on tool events at all: the session manager +// builds tool spans from the extracted call id, name, arguments, result and metadata, and drops +// the raw payload. Without this the keys would be accepted on the wire and silently discarded. +#[test] +fn pi_attempt_attribution_is_promoted_from_payload_into_metadata() { + let outcome = pi::adapt( + json!({ + "session_id": "pi-session", + "hook_event_name": "tool_call", + "tool_call_id": "call-1", + "tool_name": "read", + "input": { "path": "README.md" }, + "attempt_index": 2, + "turn_seq": 5 + }), + &HeaderMap::new(), + ); + match &outcome.events[0] { + NormalizedEvent::ToolStarted(event) => { + assert_eq!(event.metadata["attempt_index"], json!(2)); + assert_eq!(event.metadata["turn_seq"], json!(5)); + } + event => panic!("unexpected event: {event:?}"), + } +} + +// Counters only. A string here would put an untyped field into observability metadata that every +// consumer then has to defend against, and pi's own `turn_index` is deliberately never promoted +// because the gateway assigns its own to the turn scope. +#[test] +fn pi_attribution_promotion_ignores_non_numeric_and_foreign_keys() { + let outcome = pi::adapt( + json!({ + "session_id": "pi-session", + "hook_event_name": "turn_end", + "attempt_index": "one", + "turn_seq": null, + "turn_index": 4 + }), + &HeaderMap::new(), + ); + match &outcome.events[0] { + NormalizedEvent::HookMark(event) => { + assert!(event.metadata.get("attempt_index").is_none()); + assert!(event.metadata.get("turn_seq").is_none()); + assert!( + event.metadata.get("turn_index").is_none(), + "pi's resetting turn_index must not collide with the gateway's own" + ); + // The raw values stay on the payload, which is what mark events record as `data`. + assert_eq!(event.payload["turn_index"], json!(4)); + } + event => panic!("unexpected event: {event:?}"), + } +} + +// Codex and Claude Code share the classification machinery, so the two lists pi introduced must +// stay inert for them: `PreCompact`/`PostCompact` continue through the shared fallback, and +// neither harness gains a turn-start event it never sends. +#[test] +fn turn_start_and_compaction_rules_stay_inert_for_codex_and_claude() { + for outcome in [ + codex::adapt( + json!({ "session_id": "codex-session", "hook_event_name": "PreCompact" }), + &HeaderMap::new(), + ), + claude_code::adapt( + json!({ "session_id": "claude-session", "hook_event_name": "PostCompact" }), + &HeaderMap::new(), + ), + ] { + assert!( + matches!(outcome.events.as_slice(), [NormalizedEvent::Compaction(_)]), + "compaction must still classify through the shared fallback. events: {:?}", + outcome.events + ); + } + + for outcome in [ + codex::adapt( + json!({ "session_id": "codex-session", "hook_event_name": "turn_start" }), + &HeaderMap::new(), + ), + claude_code::adapt( + json!({ "session_id": "claude-session", "hook_event_name": "turn_start" }), + &HeaderMap::new(), + ), + ] { + assert!( + matches!(outcome.events.as_slice(), [NormalizedEvent::HookMark(_)]), + "only pi declares turn_start. events: {:?}", + outcome.events + ); + } +} diff --git a/crates/cli/tests/coverage/agents/alignment_tests.rs b/crates/cli/tests/coverage/agents/alignment_tests.rs index e7f810a26..286277744 100644 --- a/crates/cli/tests/coverage/agents/alignment_tests.rs +++ b/crates/cli/tests/coverage/agents/alignment_tests.rs @@ -491,7 +491,8 @@ fn route_event_through_alias_covers_all_event_variants() { NormalizedEvent::ToolStarted(event) | NormalizedEvent::ToolEnded(event) => { assert_eq!(event.subagent_id.as_deref(), Some("child")); } - NormalizedEvent::TurnEnded(_) + NormalizedEvent::TurnStarted(_) + | NormalizedEvent::TurnEnded(_) | NormalizedEvent::PromptSubmitted(_) | NormalizedEvent::Compaction(_) | NormalizedEvent::Notification(_) @@ -763,6 +764,7 @@ fn event_metadata(event: &NormalizedEvent) -> &Value { match event { NormalizedEvent::AgentStarted(event) | NormalizedEvent::AgentEnded(event) + | NormalizedEvent::TurnStarted(event) | NormalizedEvent::TurnEnded(event) | NormalizedEvent::PromptSubmitted(event) | NormalizedEvent::Compaction(event) diff --git a/crates/cli/tests/coverage/agents/coding_agent_tests.rs b/crates/cli/tests/coverage/agents/coding_agent_tests.rs index e94f78590..b27b1521f 100644 --- a/crates/cli/tests/coverage/agents/coding_agent_tests.rs +++ b/crates/cli/tests/coverage/agents/coding_agent_tests.rs @@ -22,7 +22,7 @@ fn agent_descriptors_are_complete_and_unique() { assert_eq!(CodingAgent::ClaudeCode.hook_events().len(), 14); assert_eq!(CodingAgent::Codex.hook_events().len(), 10); assert_eq!(CodingAgent::Pi.label(), "pi"); - assert_eq!(CodingAgent::Pi.hook_events().len(), 10); + assert_eq!(CodingAgent::Pi.hook_events().len(), 11); for agent in CodingAgent::ALL { let events = agent.hook_events(); assert!(events.iter().all(|event| !event.is_empty())); diff --git a/crates/cli/tests/coverage/shared/session_tests.rs b/crates/cli/tests/coverage/shared/session_tests.rs index 8531803f4..5e4347652 100644 --- a/crates/cli/tests/coverage/shared/session_tests.rs +++ b/crates/cli/tests/coverage/shared/session_tests.rs @@ -5446,3 +5446,205 @@ fn llm_start_with_content( metadata: json!({}), } } + +// -------------------------------------------------------------------------- +// pi turn boundaries and attempt attribution, end to end through the adapter +// -------------------------------------------------------------------------- + +/// Drive one pi hook payload through the real adapter and the session manager. +/// +/// Going through `pi::adapt` rather than hand-building `NormalizedEvent`s is the point: the +/// defects this covers -- turn scopes opening at the wrong event, and attribution accepted on the +/// wire and then discarded -- both live in the classification and metadata layers, not in the +/// session manager alone. +async fn apply_pi_hook(manager: &SessionManager, payload: Value) { + let outcome = crate::agents::shared::adapters::pi::adapt(payload, &HeaderMap::new()); + manager + .apply_events(&HeaderMap::new(), outcome.events) + .await + .unwrap(); +} + +#[tokio::test] +async fn pi_turn_scopes_open_at_pis_own_boundary_and_carry_attempt_attribution() { + let subscriber_name = "cli-pi-turn-boundary-test"; + let _ = deregister_subscriber(subscriber_name); + let captured = Arc::new(StdMutex::new( + Vec::<(String, Option, Value)>::new(), + )); + let events = captured.clone(); + register_subscriber( + subscriber_name, + Arc::new(move |event| { + let Some(metadata) = event.metadata() else { + return; + }; + if metadata.get("session_id").and_then(Value::as_str) != Some("pi-boundary-session") { + return; + } + events.lock().unwrap().push(( + event.name().to_string(), + event.scope_category(), + metadata.clone(), + )); + }), + ) + .unwrap(); + + let manager = SessionManager::new(session_test_config()); + let session = json!({ "session_id": "pi-boundary-session" }); + // One prompt, one attempt, one turn, one tool -- the shape every trace starts from. + for payload in [ + json!({ "hook_event_name": "session_start", "reason": "startup" }), + json!({ "hook_event_name": "agent_start", "attempt_index": 0 }), + json!({ + "hook_event_name": "turn_start", "turn_index": 0, "turn_seq": 0, "attempt_index": 0 + }), + json!({ + "hook_event_name": "tool_call", "tool_call_id": "call-1", "tool_name": "read", + "input": { "path": "README.md" }, "attempt_index": 0, "turn_seq": 0 + }), + json!({ + "hook_event_name": "tool_execution_end", "tool_call_id": "call-1", + "tool_name": "read", "status": "ok", "attempt_index": 0, "turn_seq": 0 + }), + json!({ + "hook_event_name": "turn_end", "turn_index": 0, "turn_seq": 0, "attempt_index": 0 + }), + // The run-level tail. These trail the last turn_end and used to open an empty turn. + json!({ "hook_event_name": "agent_end", "attempt_index": 0 }), + json!({ "hook_event_name": "agent_settled", "attempts": 1, "attempt_index": 0 }), + json!({ "hook_event_name": "session_shutdown", "reason": "quit" }), + ] { + let mut merged = session.clone(); + merged + .as_object_mut() + .unwrap() + .extend(payload.as_object().unwrap().clone()); + apply_pi_hook(&manager, merged).await; + } + + flush_subscribers().unwrap(); + let captured = captured.lock().unwrap(); + + let turn_starts = captured + .iter() + .filter(|(name, category, _)| name == "pi-turn" && *category == Some(ScopeCategory::Start)) + .collect::>(); + assert_eq!( + turn_starts.len(), + 1, + "pi reports one turn, so exactly one turn scope should exist; the run-level marks after \ + turn_end must not open another. captured: {:?}", + captured + .iter() + .map(|(name, category, _)| (name.as_str(), *category)) + .collect::>() + ); + let (_, _, turn_metadata) = turn_starts[0]; + assert_eq!(turn_metadata["turn_source"], json!("turn_start")); + // Attribution on the scope itself, not only on a mark inside it: this is what makes "which + // attempt did this turn belong to" answerable by walking the scope tree. + assert_eq!(turn_metadata["attempt_index"], json!(0)); + assert_eq!(turn_metadata["turn_seq"], json!(0)); + // The gateway still assigns its own turn index and never trusts pi's, which resets on re-entry. + assert_eq!(turn_metadata["turn_index"], json!(1)); + + let tool_start = captured + .iter() + .find(|(name, category, _)| name == "read" && *category == Some(ScopeCategory::Start)) + .expect("the tool span should exist"); + assert_eq!(tool_start.2["attempt_index"], json!(0)); + assert_eq!(tool_start.2["turn_seq"], json!(0)); + + assert_eq!( + captured + .iter() + .filter(|(name, category, _)| name == "pi-turn" + && *category == Some(ScopeCategory::End)) + .count(), + 1, + "the one turn should close exactly once" + ); + drop(captured); + deregister_subscriber(subscriber_name).unwrap(); +} + +// The re-entry case, which is why `turn_seq` exists: pi's `turn_index` restarts at 0 on every +// attempt, so two turns in one session both call themselves turn 0. +#[tokio::test] +async fn pi_re_entry_produces_two_turns_attributed_to_two_attempts() { + let subscriber_name = "cli-pi-reentry-attribution-test"; + let _ = deregister_subscriber(subscriber_name); + let captured = Arc::new(StdMutex::new(Vec::::new())); + let events = captured.clone(); + register_subscriber( + subscriber_name, + Arc::new(move |event| { + if event.name() != "pi-turn" || event.scope_category() != Some(ScopeCategory::Start) { + return; + } + let Some(metadata) = event.metadata() else { + return; + }; + if metadata.get("session_id").and_then(Value::as_str) != Some("pi-reentry-session") { + return; + } + events.lock().unwrap().push(metadata.clone()); + }), + ) + .unwrap(); + + let manager = SessionManager::new(session_test_config()); + let session = json!({ "session_id": "pi-reentry-session" }); + for payload in [ + json!({ "hook_event_name": "session_start", "reason": "startup" }), + json!({ "hook_event_name": "agent_start", "attempt_index": 0 }), + json!({ + "hook_event_name": "turn_start", "turn_index": 0, "turn_seq": 0, "attempt_index": 0 + }), + json!({ + "hook_event_name": "turn_end", "turn_index": 0, "turn_seq": 0, "attempt_index": 0 + }), + json!({ "hook_event_name": "agent_end", "attempt_index": 0 }), + // Re-entry: pi's turn_index collides at 0 while turn_seq keeps counting. + json!({ "hook_event_name": "agent_start", "attempt_index": 1 }), + json!({ + "hook_event_name": "turn_start", "turn_index": 0, "turn_seq": 1, "attempt_index": 1 + }), + json!({ + "hook_event_name": "turn_end", "turn_index": 0, "turn_seq": 1, "attempt_index": 1 + }), + json!({ "hook_event_name": "agent_end", "attempt_index": 1 }), + json!({ "hook_event_name": "agent_settled", "attempts": 2, "attempt_index": 1 }), + json!({ "hook_event_name": "session_shutdown", "reason": "quit" }), + ] { + let mut merged = session.clone(); + merged + .as_object_mut() + .unwrap() + .extend(payload.as_object().unwrap().clone()); + apply_pi_hook(&manager, merged).await; + } + + flush_subscribers().unwrap(); + let captured = captured.lock().unwrap(); + assert_eq!( + captured.len(), + 2, + "one turn scope per pi turn: {captured:?}" + ); + assert_eq!( + captured + .iter() + .map(|metadata| ( + metadata["attempt_index"].clone(), + metadata["turn_seq"].clone() + )) + .collect::>(), + vec![(json!(0), json!(0)), (json!(1), json!(1))], + "each turn scope must name the attempt it belonged to" + ); + drop(captured); + deregister_subscriber(subscriber_name).unwrap(); +} diff --git a/integrations/pi/README.md b/integrations/pi/README.md index 6f228e905..6eb68cf78 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -17,11 +17,20 @@ all policy and all span construction happen in the gateway. ## Status Proof of concept, tracked under -[RELAY-727](https://linear.app/nvidia/issue/RELAY-727) and -[RELAY-728](https://linear.app/nvidia/issue/RELAY-728). Verified against pi +[RELAY-727](https://linear.app/nvidia/issue/RELAY-727), +[RELAY-728](https://linear.app/nvidia/issue/RELAY-728), +[RELAY-729](https://linear.app/nvidia/issue/RELAY-729) and +[RELAY-730](https://linear.app/nvidia/issue/RELAY-730). Verified against pi `v0.84.0`. pi ships breaking changes through *minor* releases and has no major-release channel, so re-verify hook signatures before relying on them. +**Model traffic does not traverse the gateway yet** +([RELAY-732](https://linear.app/nvidia/issue/RELAY-732)). pi has no base-URL +flag and no generic environment override — it resolves `baseUrl` per model from +a generated catalog — so redirection needs this extension to register a +gateway-backed provider, and nothing does that yet. Until it lands there are no +LLM spans and no model-call enforcement, only tool and turn activity. + ## Usage Start the gateway, then load the extension: @@ -74,11 +83,33 @@ Two shapes make a naive mapping wrong. **Agent-run re-entry.** One prompt can re-enter the agent run several times (provider retry, post-compaction, queued follow-up), and pi's `turnIndex` resets to 0 each time. The extension-facing `agent_end` carries no `willRetry` marker, -so a retry cannot be detected there; `agent_settled` is the only event that fires -exactly once per logical run. The gateway's own model is flat -(session -> turn -> tool) and assigns its own monotonic turn index, so the -extension sends `attempt_index` and a session-monotonic `turn_seq` as metadata — -they are the only way to recover which attempt a turn belonged to. +so a retry cannot be detected there — `session_before_compact` is the one hook +that announces one in advance, and only for the compaction case; +`agent_settled` is the only event that fires exactly once per logical run. The +gateway's own model is flat (session -> turn -> tool) and assigns its own +monotonic turn index, so the extension sends `attempt_index` and a +session-monotonic `turn_seq` on every attributable hook — they are the only way +to recover which attempt a turn or tool call belonged to. + +Both travel as payload keys, and the gateway's pi extractor promotes them into +each event's **metadata**. That promotion is not cosmetic: mark events record +the raw payload as their `data`, but tool spans are built from the extracted +call id, name, arguments, result and metadata and discard the payload entirely, +so without it the two keys would be accepted on the wire and then dropped. Read +them from `metadata` on scopes and spans, and from either place on marks. + +The consequence is worth stating plainly: **re-entry is not nested.** The +gateway model stays flat and gains no attempt level, because that model is +shared with Codex and Claude Code, which have no equivalent concept. Two +attempts of one prompt appear as more turns under one session, distinguished by +`attempt_index` — not as two subtrees. + +**Known limitation.** The counters live in the extension factory's closure and +pi re-runs the factory on `/reload` with `moduleCache: false`, while the session +id stays the same. `turn_seq` therefore restarts at 0 and can repeat within one +session — it orders turns within a runtime, not strictly within a session. +Rebuilding it would mean replaying the session or moving the counter into the +gateway; neither is worth it here. **Concurrent tools.** pi preflights sibling calls sequentially then executes them concurrently, so `tool_execution_end` arrives out of submission order. All @@ -86,18 +117,26 @@ per-call state is keyed by `toolCallId`, the only correlator pi provides. | pi hook | Forwarded as | Note | |---|---|---| -| `session_start` / `session_shutdown` | session boundary | **Not** `agent_start`/`agent_end` — those repeat on re-entry | -| `agent_start` / `agent_end` | attempt markers | Carry `attempt_index`; not a run boundary | -| `agent_settled` | logical run boundary | Fires exactly once, from a `finally` | -| `turn_start` | turn boundary (open) | Carries `turn_index`, `turn_seq` and `attempt_index` | -| `turn_end` | turn boundary (close) | Carries `turn_index` only | -| `tool_call` | tool start, and the gate | The only blocking hook | -| `tool_execution_end` | tool end | For **every** outcome, including blocked | -| `tool_execution_start` | *not forwarded* | Fires before validation and for calls that never execute | +| `session_start` / `session_shutdown` | session boundary | **Not** `agent_start`/`agent_end` — those repeat on re-entry. `session_shutdown` is ignored for `reason: "reload"`, which continues the same session | +| `agent_start` / `agent_end` | run-level marks | Carry `attempt_index`; not a run boundary. Recorded on the session scope, not inside a turn | +| `agent_settled` | run-level mark | Fires exactly once, from a `finally`. Carries `attempts` (the count) and `attempt_index` (the last one) | +| `turn_start` | turn scope **open** | Carries `turn_index`, `turn_seq`, `attempt_index` | +| `turn_end` | turn scope **close** | Carries `turn_index`, `turn_seq`, `attempt_index` | +| `session_before_compact` | mark | Announced, not done, and cancellable by a later extension. Carries `reason`, `will_retry`, `tokens_before` | +| `session_compact` | compaction | The completed compaction, which the runtime treats as proof the context was rebuilt | +| `tool_call` | tool start, and the gate | The only blocking hook. Carries `attempt_index`, `turn_seq` | +| `tool_execution_end` | tool end | For **every** outcome, including blocked. Carries `attempt_index`, `turn_seq` | +| `tool_execution_start` | *not forwarded* | Registered, but only to remember a tool name for the matching end: it fires before validation and for calls that never execute | `tool_result` is deliberately unused: it does not fire for blocked calls, and in the parallel path it fires *before* `tool_execution_end`. +Because pi reports both ends of a turn, the gateway never invents one for it. A +mark arriving between turns — the `agent_end` / `agent_settled` tail of a run — +is recorded on the session scope rather than opening an empty turn to hold it. +Codex and Claude Code report only `Stop`, so their turns stay lazily opened by +the first event of the turn. + ## Development ```bash diff --git a/integrations/pi/index.ts b/integrations/pi/index.ts index e1230c8c1..71d3d29b0 100644 --- a/integrations/pi/index.ts +++ b/integrations/pi/index.ts @@ -23,10 +23,14 @@ * (provider retry, post-compaction, queued follow-up), and `turnIndex` resets * to 0 each time, so turn indices collide within one prompt. The extension * `agent_end` payload carries no `willRetry` marker, so a retry cannot be - * detected there. `agent_settled` is the only event that fires exactly once - * per logical run. Both an attempt counter and a session-monotonic turn - * sequence are therefore sent as metadata, because the gateway's own model is - * flat (session -> turn -> tool) and cannot express the nesting. + * detected there -- `session_before_compact` is the one hook that announces + * one in advance, and only for the compaction case. `agent_settled` is the + * only event that fires exactly once per logical run. Every attributable + * hook therefore carries an attempt counter and a session-monotonic turn + * sequence, because the gateway's own model is flat (session -> turn -> + * tool) and cannot express the nesting. They travel as payload keys and the + * gateway promotes them into event metadata, which is what makes them + * survive on tool spans. * 2. *Concurrent tools.* pi preflights sibling calls sequentially then executes * them concurrently, so `tool_execution_end` arrives out of submission order. * All per-call state is keyed by `toolCallId`, which is the only correlator @@ -52,6 +56,8 @@ import type { AgentStartEvent, ExtensionAPI, ExtensionContext, + SessionBeforeCompactEvent, + SessionCompactEvent, SessionShutdownEvent, SessionStartEvent, ToolCallEvent, @@ -120,6 +126,26 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { void enqueue(() => postAndForget(active, payload)); }; + /** + * The attempt and turn a hook belongs to. + * + * Both counters hold the *next* value -- they are incremented as soon as the + * event that opens their span is forwarded -- so the live attempt and turn + * are one behind. The gateway promotes these two keys out of the payload and + * into event metadata, which is the only reason they survive on tool events: + * tool spans are built from the extracted call id, name, arguments, result + * and metadata, and the raw payload is discarded. + * + * Sent on every hook that can be attributed. Without it, `tool_call` and + * `tool_execution_end` are recoverable to an attempt only by reading + * surrounding events in arrival order, which stops working the moment two + * attempts are in flight. + */ + const attribution = (): { attempt_index: number; turn_seq: number } => ({ + attempt_index: Math.max(0, attemptIndex - 1), + turn_seq: Math.max(0, turnSeq - 1), + }); + // --------------------------------------------------------------------------- // Session lifecycle // @@ -184,24 +210,80 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { }); pi.on('agent_settled', async (_event: AgentSettledEvent, ctx: ExtensionContext) => { - emit(ctx, { hook_event_name: 'agent_settled', attempts: attemptIndex }); + emit(ctx, { + hook_event_name: 'agent_settled', + // Two different facts, both wanted: `attempts` is how many attempts the + // run took, `attempt_index` is the last of them. Sending only the count + // made this the one hook a consumer filtering on `attempt_index` missed. + attempts: attemptIndex, + ...attribution(), + }); attemptIndex = 0; }); pi.on('turn_start', async (event: TurnStartEvent, ctx: ExtensionContext) => { + const seq = turnSeq; + turnSeq += 1; emit(ctx, { hook_event_name: 'turn_start', // pi's turn_index resets to 0 on re-entry; turn_seq does not, so a // consumer can still order turns across the whole session. turn_index: event.turnIndex, - turn_seq: turnSeq, + turn_seq: seq, attempt_index: Math.max(0, attemptIndex - 1), }); - turnSeq += 1; }); pi.on('turn_end', async (event: TurnEndEvent, ctx: ExtensionContext) => { - emit(ctx, { hook_event_name: 'turn_end', turn_index: event.turnIndex }); + emit(ctx, { + hook_event_name: 'turn_end', + // pi carries turn_index on the close but not turn_seq, so the close could + // not be matched to its own open across a re-entry, where turn_index 0 + // appears once per attempt. + turn_index: event.turnIndex, + ...attribution(), + }); + }); + + // --------------------------------------------------------------------------- + // Compaction + // + // Only `session_compact` is a compaction *event* to the gateway; the runtime + // treats one as proof the context was rebuilt and marks the agent fresh so + // the next model call records full context rather than a delta. That effect + // is latent until pi's model traffic is routed through the gateway, but the + // boundary is recorded now either way. + // --------------------------------------------------------------------------- + + /** + * Announced, not yet done, and cancellable by any extension loading after + * this one -- so it is forwarded as a mark. Its `willRetry` is the only + * advance notice pi gives an extension that the agent run is about to + * re-enter; `agent_end` carries no such marker. + * + * Returns nothing on purpose: a returned object is how pi's API spells + * "cancel this compaction, or replace its result". + */ + pi.on('session_before_compact', async (event: SessionBeforeCompactEvent, ctx) => { + emit(ctx, { + hook_event_name: 'session_before_compact', + reason: event.reason, + will_retry: event.willRetry, + tokens_before: event.preparation?.tokensBefore, + is_split_turn: event.preparation?.isSplitTurn, + ...attribution(), + }); + }); + + pi.on('session_compact', async (event: SessionCompactEvent, ctx) => { + emit(ctx, { + hook_event_name: 'session_compact', + reason: event.reason, + will_retry: event.willRetry, + from_extension: event.fromExtension, + tokens_before: event.compactionEntry?.tokensBefore, + ...attribution(), + }); }); // --------------------------------------------------------------------------- @@ -241,6 +323,7 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { tool_call_id: event.toolCallId, tool_name: event.toolName, input: event.input, + ...attribution(), }), ); @@ -271,6 +354,7 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { tool_name: toolName, result: summarize(event.result, event.isError), status: event.isError ? 'error' : 'ok', + ...attribution(), }); }); } diff --git a/integrations/pi/src/pi-hook-types.ts b/integrations/pi/src/pi-hook-types.ts index 4e578852f..a5b130637 100644 --- a/integrations/pi/src/pi-hook-types.ts +++ b/integrations/pi/src/pi-hook-types.ts @@ -62,6 +62,33 @@ export type SessionShutdownEvent = { targetSessionFile?: string; }; +/** + * Fired *before* context compaction, and cancellable. + * + * `willRetry` is the one place pi tells an extension that a re-entry is coming: + * the extension-facing `agent_end` carries no such marker. `preparation` also + * carries the pre-compaction token count and whether the cut lands mid-turn. + * + * A handler must return `undefined` here. Returning an object is how pi's API + * spells "cancel this compaction, or replace its result", so an accidental + * return value from an observability hook would change pi's behaviour. + */ +export type SessionBeforeCompactEvent = { + type: 'session_before_compact'; + reason: 'manual' | 'threshold' | 'overflow'; + willRetry: boolean; + preparation?: { tokensBefore?: number; isSplitTurn?: boolean }; +}; + +/** Fired after context compaction has actually happened. Not cancellable. */ +export type SessionCompactEvent = { + type: 'session_compact'; + reason: 'manual' | 'threshold' | 'overflow'; + willRetry: boolean; + fromExtension: boolean; + compactionEntry?: { tokensBefore?: number }; +}; + /** * Fired when a tool starts executing. * @@ -121,6 +148,8 @@ export type ExtensionHandler = ( export type ExtensionAPI = { on(event: 'session_start', handler: ExtensionHandler): void; on(event: 'session_shutdown', handler: ExtensionHandler): void; + on(event: 'session_before_compact', handler: ExtensionHandler): void; + on(event: 'session_compact', handler: ExtensionHandler): void; on(event: 'agent_start', handler: ExtensionHandler): void; on(event: 'agent_end', handler: ExtensionHandler): void; on(event: 'agent_settled', handler: ExtensionHandler): void; diff --git a/integrations/pi/test/lifecycle.test.mjs b/integrations/pi/test/lifecycle.test.mjs index 7f642a1c4..223bf5b55 100644 --- a/integrations/pi/test/lifecycle.test.mjs +++ b/integrations/pi/test/lifecycle.test.mjs @@ -220,3 +220,168 @@ describe('session_shutdown reason', () => { assert.equal(named(ctx.posts, 'session_shutdown')[0].target_session_file, '/s/next.jsonl'); }); }); + +describe('attribution on every hook that has one', () => { + let ctx; + + before(async () => { + ctx = stubGateway(); + await new Promise((r) => ctx.server.listen(0, '127.0.0.1', r)); + process.env.NEMO_RELAY_PI_GATEWAY_URL = `http://127.0.0.1:${ctx.server.address().port}`; + }); + + after(() => { + ctx.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + }); + + beforeEach(() => { + ctx.posts.length = 0; + }); + + // Tool hooks used to carry no attribution at all, which made "which attempt ran this tool" + // answerable only by reading surrounding events in arrival order -- and that stops working the + // moment two attempts are in flight. + it('attributes tool hooks to the attempt and turn that ran them', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 1 }); + await fire('turn_end', { turnIndex: 0 }); + await fire('agent_end', { messages: [] }); + // Second attempt: pi's turn_index is 0 again, so only turn_seq separates the two turns. + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 2 }); + await fire('tool_execution_start', { toolCallId: 'c1', toolName: 'read', args: {} }); + await fire('tool_call', { toolCallId: 'c1', toolName: 'read', input: { path: 'a.txt' } }); + await fire('tool_execution_end', { + toolCallId: 'c1', + toolName: 'read', + result: 'ok', + isError: false, + }); + await fire('turn_end', { turnIndex: 0 }); + await fire('session_shutdown', { reason: 'quit' }); + + for (const name of ['tool_call', 'tool_execution_end']) { + const post = named(ctx.posts, name)[0]; + assert.equal(post.attempt_index, 1, `${name} attempt_index`); + assert.equal(post.turn_seq, 1, `${name} turn_seq`); + } + }); + + // pi carries turn_index on the close but not turn_seq, so without this the close could not be + // matched to its own open across a re-entry, where turn_index 0 appears once per attempt. + it('gives turn_end the same turn_seq its turn_start announced', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 1 }); + await fire('turn_end', { turnIndex: 0 }); + await fire('turn_start', { turnIndex: 1, timestamp: 2 }); + await fire('turn_end', { turnIndex: 1 }); + await fire('session_shutdown', { reason: 'quit' }); + + assert.deepEqual( + named(ctx.posts, 'turn_end').map((p) => [p.turn_index, p.turn_seq, p.attempt_index]), + [ + [0, 0, 0], + [1, 1, 0], + ], + ); + }); + + // `attempts` is how many attempts the run took; `attempt_index` is the last of them. Sending + // only the count made this the one hook a consumer filtering on attempt_index missed. + it('sends agent_settled the attempt count and the last attempt index', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('agent_start'); + await fire('agent_end', { messages: [] }); + await fire('agent_start'); + await fire('agent_end', { messages: [] }); + await fire('agent_settled'); + await fire('session_shutdown', { reason: 'quit' }); + + const settled = named(ctx.posts, 'agent_settled')[0]; + assert.equal(settled.attempts, 2); + assert.equal(settled.attempt_index, 1); + }); +}); + +describe('compaction', () => { + let ctx; + + before(async () => { + ctx = stubGateway(); + await new Promise((r) => ctx.server.listen(0, '127.0.0.1', r)); + process.env.NEMO_RELAY_PI_GATEWAY_URL = `http://127.0.0.1:${ctx.server.address().port}`; + }); + + after(() => { + ctx.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + }); + + beforeEach(() => { + ctx.posts.length = 0; + }); + + it('forwards both the announcement and the completion, with the token count', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 1 }); + await fire('session_before_compact', { + reason: 'threshold', + willRetry: false, + preparation: { tokensBefore: 120_000, isSplitTurn: true }, + }); + await fire('session_compact', { + reason: 'threshold', + willRetry: false, + fromExtension: false, + compactionEntry: { tokensBefore: 120_000 }, + }); + await fire('session_shutdown', { reason: 'quit' }); + + const announced = named(ctx.posts, 'session_before_compact')[0]; + assert.equal(announced.reason, 'threshold'); + assert.equal(announced.will_retry, false); + assert.equal(announced.tokens_before, 120_000); + assert.equal(announced.is_split_turn, true); + assert.equal(announced.attempt_index, 0); + assert.equal(announced.turn_seq, 0); + + const completed = named(ctx.posts, 'session_compact')[0]; + assert.equal(completed.reason, 'threshold'); + assert.equal(completed.from_extension, false); + assert.equal(completed.tokens_before, 120_000); + }); + + // pi spells "cancel this compaction, or replace its result" as a returned object, so an + // observability handler that returned anything would change pi's behaviour. + it('never returns a value that could cancel or replace pi compaction', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + assert.equal( + await fire('session_before_compact', { reason: 'manual', willRetry: false }), + undefined, + ); + assert.equal( + await fire('session_compact', { reason: 'manual', willRetry: false, fromExtension: false }), + undefined, + ); + await fire('session_shutdown', { reason: 'quit' }); + }); + + // `willRetry` on the announcement is the only advance notice pi gives an extension that the + // agent run is about to re-enter; `agent_end` carries no such marker. + it('carries the retry marker that agent_end lacks', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('session_before_compact', { reason: 'overflow', willRetry: true }); + await fire('session_shutdown', { reason: 'quit' }); + assert.equal(named(ctx.posts, 'session_before_compact')[0].will_retry, true); + }); +}); From 839bccbf1593eacccdcadb933c2540c8418a5bc1 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 15:25:16 -0700 Subject: [PATCH 07/41] ci(pi): give the pi integration CI, a just recipe, and docs integrations/pi/ had zero CI: nothing ran its tests, nothing typechecked it, and it was not an npm workspace, so its package scripts were unreachable from the repo root. Adds it as a workspace member and a `test-pi` recipe, threaded through the same four layers OpenClaw uses: a `pi:` path filter, a `run_pi` output from ci_changes, an input on ci_node, and the pass-through in ci.yaml. The filter also covers crates/cli/src/agents/pi/ and the shared adapter, because the extension and the gateway share one wire contract and a change to either can break the other. `run_node` now also fires on a pi-only change, or the job that hosts the step would never start. Unlike test-openclaw, the recipe does not build the Node binding first: the pi extension is a sidecar HTTP client and loads no native addon. Docs: adds docs/nemo-relay-cli/pi.mdx and lists pi in the four places that enumerate agents -- the CLI about page, basic usage, the support matrix, and the root README. The page is explicit about what is not there: no persistent install because pi has no plugin marketplace, no LLM spans because pi's model traffic does not traverse the gateway, and no subagent representation. Two claims were corrected against the binary while writing the page. There is no `nemo-relay pi` shortcut subcommand -- pi runs through `nemo-relay run --agent pi` -- and NEMO_RELAY_PI_EXTENSION is required rather than optional, because pi extensions live in the user's own configuration directories and there is no Relay-managed location to fall back on. just docs-linkcheck passes. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- .github/ci-path-filters.yml | 8 ++ .github/workflows/ci.yaml | 1 + .github/workflows/ci_changes.yml | 6 +- .github/workflows/ci_node.yml | 10 ++ README.md | 1 + docs/nemo-relay-cli/about.mdx | 4 + docs/nemo-relay-cli/basic-usage.mdx | 10 ++ docs/nemo-relay-cli/pi.mdx | 216 ++++++++++++++++++++++++++++ docs/reference/support-matrix.mdx | 4 +- justfile | 18 ++- package-lock.json | 11 +- package.json | 3 +- 12 files changed, 287 insertions(+), 5 deletions(-) create mode 100644 docs/nemo-relay-cli/pi.mdx diff --git a/.github/ci-path-filters.yml b/.github/ci-path-filters.yml index d0efde810..e20d685cf 100644 --- a/.github/ci-path-filters.yml +++ b/.github/ci-path-filters.yml @@ -190,6 +190,14 @@ node: openclaw: - 'integrations/openclaw/**' +# The pi extension is a hook client for the CLI gateway, so its contract is +# shared with the Rust adapter and route -- a change on either side can break +# the other. +pi: + - 'crates/cli/src/agents/pi/**' + - 'crates/cli/src/agents/shared/adapters.rs' + - 'integrations/pi/**' + python: - 'crates/python/Cargo.toml' - 'crates/python/src/**' diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ae107c9c1..03d0fa614 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -175,6 +175,7 @@ jobs: ref_name: ${{ github.ref_name }} run_package: ${{ needs.ci_changes.outputs.run_node_package == 'true' }} run_openclaw: ${{ needs.ci_changes.outputs.run_openclaw == 'true' }} + run_pi: ${{ needs.ci_changes.outputs.run_pi == 'true' }} ci_python: name: Python diff --git a/.github/workflows/ci_changes.yml b/.github/workflows/ci_changes.yml index 0d7ea81c7..e6eb7b43b 100644 --- a/.github/workflows/ci_changes.yml +++ b/.github/workflows/ci_changes.yml @@ -48,6 +48,9 @@ on: run_openclaw: description: 'Whether OpenClaw integration jobs should run' value: ${{ jobs.changes.outputs.run_openclaw }} + run_pi: + description: 'Whether pi integration jobs should run' + value: ${{ jobs.changes.outputs.run_pi }} run_python: description: 'Whether Python jobs should run' value: ${{ jobs.changes.outputs.run_python }} @@ -77,9 +80,10 @@ jobs: run_docs: ${{ (inputs.full_ci || startsWith(inputs.ref_name, 'pull-request/')) && (inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.docs == 'true') }} run_go: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.go == 'true' }} run_installer: ${{ inputs.full_ci || steps.filter.outputs.installer == 'true' }} - run_node: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.node == 'true' || steps.filter.outputs.openclaw == 'true' }} + run_node: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.node == 'true' || steps.filter.outputs.openclaw == 'true' || steps.filter.outputs.pi == 'true' }} run_node_package: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.node_package == 'true' }} run_openclaw: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.node == 'true' || steps.filter.outputs.openclaw == 'true' }} + run_pi: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.node == 'true' || steps.filter.outputs.pi == 'true' }} run_python: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.python == 'true' }} run_python_integration_langchain: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.python_integration_langchain == 'true' }} run_python_package: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.python_package == 'true' }} diff --git a/.github/workflows/ci_node.yml b/.github/workflows/ci_node.yml index c2c5581ef..5d5401cd2 100644 --- a/.github/workflows/ci_node.yml +++ b/.github/workflows/ci_node.yml @@ -24,6 +24,11 @@ on: required: false default: false type: boolean + run_pi: + description: 'Whether to run pi integration checks' + required: false + default: false + type: boolean secrets: CODECOV_TOKEN: required: false @@ -122,6 +127,11 @@ jobs: working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} run: just --set ci true test-openclaw + - name: Run pi integration checks + if: ${{ inputs.run_pi }} + working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + run: just --set ci true test-pi + - name: Upload Node coverage to Codecov uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6 if: ${{ !startsWith(matrix.platform, 'windows') }} diff --git a/README.md b/README.md index 3a033090d..9ba09612d 100644 --- a/README.md +++ b/README.md @@ -315,6 +315,7 @@ coverage. |:--|:--:|:--:|:--:|:--| | Claude Code | Yes | Yes | Partial | Hook forwarding, pre-tool blocking, and gateway-routed LLM observability are supported. | | Codex | Yes | Yes | Partial | Persistent install verifies the exact plugin hooks. Each `Stop` finalizes a turn snapshot; the supported generated schema does not install `SessionEnd`. | +| pi | Partial | Partial | No | Proof of concept. A Relay-authored pi extension forwards tool and turn activity and gates tool calls; pi's model traffic does not traverse the gateway, so there are no LLM spans. | | Hermes Agent | Yes | Yes | Partial | NeMo Relay is built into Hermes Agent, and Hermes Agent understands NeMo Relay plugin configurations. No separate observability plugin or Relay CLI setup is required. | ### Public API Integrations diff --git a/docs/nemo-relay-cli/about.mdx b/docs/nemo-relay-cli/about.mdx index ae3768a5d..5378627b8 100644 --- a/docs/nemo-relay-cli/about.mdx +++ b/docs/nemo-relay-cli/about.mdx @@ -36,6 +36,7 @@ Use these guides when you need to: - Route model-provider traffic through the local NeMo Relay gateway. - Install persistent Claude Code or Codex integrations without wrapping the agent command. +- Gate pi tool calls on a Relay policy from inside a pi extension. - Validate and install model pricing catalog sources for local cost estimates. - Export local sessions to Agent Trajectory Interchange Format (ATIF), Agent Trajectory Observability Format (ATOF) JSONL, OpenTelemetry, or @@ -57,6 +58,7 @@ controls. | --- | --- | --- | --- | --- | | Claude Code | Yes | Yes | Partial | Pre-tool hook responses are supported. LLM optimization uses gateway-routed traffic; full coverage depends on loaded Claude Code hooks. | | Codex | Yes | Yes | Partial | Persistent install verifies all 10 hooks in the supported schema. Each `Stop` finalizes a turn snapshot because the plugin schema does not expose `SessionEnd`. | +| pi | Partial | Partial | No | Proof of concept. Tool and turn activity are captured through a Relay-authored pi extension, and tool calls can be blocked. pi's model traffic does not traverse the gateway, so there are no LLM spans and no model-call enforcement. | For minimum agent versions, platform support, and current limitations, refer to the [Support Matrix](/reference/support-matrix). @@ -74,6 +76,8 @@ Use these guide links to move from CLI setup into agent-specific instructions. application modes. - [Codex](/nemo-relay-cli/codex) covers transparent Codex CLI runs, local GUI/app caveats, model provider routing, and remote-task limits. +- [pi](/nemo-relay-cli/pi) covers the extension-based pi integration, tool + gating, agent-run attribution, and the model-redirection gap. Start with [Basic Usage](/nemo-relay-cli/basic-usage), then use the guide for the coding agent that you want to observe. diff --git a/docs/nemo-relay-cli/basic-usage.mdx b/docs/nemo-relay-cli/basic-usage.mdx index d7f22ca8d..8f0b43267 100644 --- a/docs/nemo-relay-cli/basic-usage.mdx +++ b/docs/nemo-relay-cli/basic-usage.mdx @@ -25,6 +25,9 @@ the payload in a shared gateway envelope. - `POST /hooks/claude-code` accepts Claude Code hook JSON and returns Claude-compatible fields such as `continue` and permission decisions when the hook event supports them. +- `POST /hooks/pi` accepts the hook JSON posted by the NeMo Relay pi extension. + It returns an empty object for an allowed call and HTTP 403 with + `error.type = "nemo_relay_guardrail_rejected"` when a guardrail blocks one. When a hook closes a turn, subagent, or session scope, Relay returns the hook response after subscribers finish processing that scope-end event. This makes @@ -474,6 +477,11 @@ calling NeMo Relay APIs. agent-thought hook. These hints are not emitted as NeMo Relay events. - Compaction, notification, and unknown hook events become mark events under the active session scope. +- Harnesses that report the *start* of a turn as well as its end open the turn + scope at that signal instead of lazily. pi does; Claude Code and Codex report + only `Stop`, so their turns stay lazily opened by the first event of the turn. + For a harness with an explicit turn start, a mark that arrives between turns + is recorded on the session scope rather than opening a turn to hold it. - Gateway requests emit NeMo Relay LLM start and end events under the active session scope. Before each LLM start, the gateway uses explicit subagent headers, pending hints, shared conversation/generation/request identifiers, @@ -515,6 +523,7 @@ Generated hook bundles subscribe to the events needed for that mapping: | --- | --- | --- | | Claude Code | `UserPromptSubmit`, `Stop` | `SessionStart`, `SessionEnd`, `UserPromptExpansion`, `SubagentStart`, `SubagentStop`, `PreToolUse`, `PostToolUse`, `PostToolUseFailure`, `PermissionRequest`, `Notification`, `PreCompact`, `PostCompact` | | Codex | `UserPromptSubmit`, `Stop` | `SessionStart`, `SubagentStart`, `SubagentStop`, `PreToolUse`, `PostToolUse`, `PermissionRequest`, `PreCompact`, `PostCompact` | +| pi | none yet — pi's model traffic does not reach the gateway | `session_start`, `session_shutdown`, `agent_start`, `agent_end`, `agent_settled`, `turn_start`, `turn_end`, `session_before_compact`, `session_compact`, `tool_call`, `tool_execution_end` | ## Hook Forwarding @@ -564,6 +573,7 @@ application-mode caveats. - [Claude Code](/nemo-relay-cli/claude-code) - [Codex](/nemo-relay-cli/codex) +- [pi](/nemo-relay-cli/pi) - [Coding Agent Installation](/nemo-relay-cli/plugin-installation) Each guide covers transparent run setup, gateway routing, hook smoke tests, diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx new file mode 100644 index 000000000..1b062f4d8 --- /dev/null +++ b/docs/nemo-relay-cli/pi.mdx @@ -0,0 +1,216 @@ +--- +title: "pi" +description: "" +position: 6 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + + +Use this guide to observe local [pi](https://github.com/earendil-works/pi) +sessions and to gate pi tool calls on a NeMo Relay policy. The pi integration is +a proof of concept: tool and turn activity reach Relay, and LLM traffic does not +yet, so there are no LLM spans and no model-call enforcement. + + +pi ships breaking changes through *minor* releases and has no major-release +channel. The integration is verified against pi `v0.84.0`. Re-verify hook +signatures after a pi upgrade; a silent shape change appears as missing spans, +not as an error. + + +## How It Differs From Codex and Claude Code + +pi has no native hook-configuration file, and its external event stream is +observation-only, so hook calls cannot be injected from outside the process. +They must originate inside a pi *extension*. Relay ships one at +`integrations/pi/`, and it is a thin HTTP client: it forwards pi's lifecycle to +`POST /hooks/pi` and gates tool calls on the gateway's verdict. All policy and +all span construction stay in the gateway. + +Three consequences follow from that shape: + +- There is no persistent plugin install. pi has no plugin marketplace — no + manifest, no `pi plugin` verb, and no MCP client for a plugin-owned server to + serve — so `nemo-relay install pi` is unsupported and says so. Install the + extension with `pi install ` or place it in an auto-discovered + directory (`~/.pi/agent/extensions/`, `.pi/extensions/`). +- The extension pays a round trip per gated tool call. pi awaits extension + handlers on its critical path, so a blocking hook is synchronous by + construction. Observability-only hooks are queued rather than awaited and + drained at shutdown, so they do not charge that path. +- Model traffic is not redirected. pi resolves `baseUrl` per model from a + generated catalog and has no base-URL flag or generic environment override, + so redirection requires the extension to register a gateway-backed provider. + That work is not done yet. + +## Requirements + +Install pi 0.84.0 or newer, and confirm the NeMo Relay extension is reachable: + +```bash +nemo-relay doctor pi +``` + +## Transparent Run + +Use the wrapper for no-install local observability. pi has no `nemo-relay pi` +shortcut of its own, unlike Claude Code and Codex; run it through `run`: + +```bash +nemo-relay run --agent pi +``` + +The wrapper starts a gateway on a dynamic `127.0.0.1` port, loads the NeMo Relay +extension with `pi -e `, and passes the gateway URL to it through +`NEMO_RELAY_PI_GATEWAY_URL`. `pi -e` is the reliable loader here: it is +trust-ungated, loads before extension discovery, and survives `--no-extensions`, +so a launched session is instrumented regardless of the user's own pi +configuration. + +Set `NEMO_RELAY_PI_EXTENSION` to the extension entry point. Unlike Claude Code +and Codex, there is no Relay-managed install location to fall back on, because +pi extensions live in the user's own configuration directories. Launch fails +with that instruction when the extension cannot be located. + +Inspect what would be launched without starting pi: + +```bash +nemo-relay run --dry-run --print --agent pi +``` + +## Standalone Gateway + +Run a long-lived gateway and point pi at it by hand: + +```bash +nemo-relay --bind 127.0.0.1:4040 & +NEMO_RELAY_PI_GATEWAY_URL=http://127.0.0.1:4040 \ + pi -e integrations/pi/index.ts +``` + +| Variable | Default | Meaning | +| --- | --- | --- | +| `NEMO_RELAY_PI_GATEWAY_URL` | `http://127.0.0.1:4040` | Gateway base URL | +| `NEMO_RELAY_PI_TIMEOUT_MS` | `5000` | Per-request timeout | +| `NEMO_RELAY_PI_FAIL` | `open` | Set to `closed` to block tool calls when the gateway is unreachable | + + +Run headless pi with `< /dev/null`. pi drains piped standard input before the +session starts and returns early only for a TTY, so under an automated harness +it can block indefinitely before any hook fires. + + +## Tool Gating + +`tool_call` is the only pi hook that can block, and for model-invoked tools it +is the only pre-execution decision point that sees arguments: pi applies +`--tools`, `--exclude-tools`, `--no-tools`, and the runtime `setActiveTools` at +tool-registry construction, never per call. + +A guardrail rejection surfaces as HTTP 403 with +`error.type = "nemo_relay_guardrail_rejected"` and the guardrail's own words in +`error.reason`. The extension turns that into pi's `{block, reason}`. + +| Gateway response | Extension behavior | +| --- | --- | +| 2xx | Allow | +| 403 with the guardrail marker | Block, using `error.reason` | +| 403 without that marker | Fault — an authorization failure is not a policy decision | +| Other status, timeout, unreachable | Fault, resolved by `NEMO_RELAY_PI_FAIL` | + +pi passes the block reason to the model verbatim, with no framing. Write +guardrail reasons as guidance rather than as error codes: a reason that names a +safe alternative produces a model that adapts, and a bare denial produces one +that gives up. + +The default is fail-open, so an unreachable gateway does not brick the agent. +`NEMO_RELAY_PI_FAIL=closed` opts in to blocking, and the block reason then says +explicitly that it is an infrastructure fault rather than a judgment about the +request. + +## Captured Events + +The extension posts 11 hooks. pi reports both ends of a turn, so Relay opens the +turn scope at pi's own `turn_start` instead of inferring one, and a mark that +arrives between turns is recorded on the session scope rather than opening an +empty turn to hold it. + +| pi hook | Relay lifecycle | +| --- | --- | +| `session_start` / `session_shutdown` | Session scope. `session_shutdown` is ignored for `reason: "reload"`, which continues the same session | +| `agent_start` / `agent_end` / `agent_settled` | Marks on the session scope | +| `turn_start` / `turn_end` | Turn scope open and close | +| `session_before_compact` | Mark. The compaction is announced, not yet done, and a later extension can still cancel it | +| `session_compact` | Canonical `compaction` mark | +| `tool_call` | Tool span start, and the gate | +| `tool_execution_end` | Tool span end, for every outcome including blocked | + +pi's `tool_execution_start` is deliberately not forwarded: it fires before +argument validation and also for calls that never execute. `tool_result` is not +forwarded either, because it does not fire at all for blocked calls and in the +parallel path it fires before `tool_execution_end`. + +A blocked call still produces a well-formed tool span. The guardrail rejects +before the span opens, but pi fires `tool_execution_end` for blocked calls too, +and the gateway synthesizes the missing pair, tagged +`tool_correlation_status: "agent_fallback"`. + +## Agent-Run Attribution + +One pi prompt can re-enter the agent run several times — provider retry, +post-compaction recovery, or a queued follow-up — and pi's own `turnIndex` +resets to 0 on each re-entry, so turn indices collide within one prompt. + +Relay's session model is flat (session → turn → tool) and is shared with Codex +and Claude Code, so it gains no attempt level for pi. **Re-entry is therefore +not nested**: two attempts of one prompt appear as more turns under one session. +To keep them distinguishable, the extension sends two counters on every +attributable hook, and the gateway promotes both into event metadata: + +- `attempt_index` — which agent-run attempt this event belongs to. +- `turn_seq` — a session-monotonic turn counter, where pi's `turn_index` resets. + +Read them from `metadata` on turn scopes and tool spans, and from either +`metadata` or the mark's `data` on marks. The turn scope's own `turn_index` is +assigned by Relay and is monotonic; pi's colliding value stays in the payload. + + +`turn_seq` can repeat within one session. The counters live in the extension +runtime, and pi rebuilds that runtime on `/reload` while the session id stays +the same, so the counter restarts at 0. It orders turns within a runtime, not +strictly within a session. + + +Subagents are not represented. pi has no nested-agent hook of its own, and a +child pi process running this extension posts under its own session id, so it +appears as an unrelated session rather than as a subagent. + +## Smoke Test + +Check hook forwarding directly. A 200 is an allow; a 403 with the guardrail +marker is a block: + +```bash +curl -f http://127.0.0.1:4040/healthz +curl -s -w '\nHTTP:%{http_code}\n' -X POST http://127.0.0.1:4040/hooks/pi \ + -H 'content-type: application/json' \ + -d '{"hook_event_name":"tool_call","session_id":"smoke-pi","tool_call_id":"c1","tool_name":"read","input":{"path":"README.md"}}' +``` + +## Verify Export + +Complete a pi turn, then confirm the exporter output. With an ATOF file sink +configured in `plugins.toml`, a single-tool turn produces a `pi` session scope, +one `pi-turn` scope with `turn_source: turn_start`, and the tool span nested +under it. + +If turn scopes are missing entirely, the extension is not loading. pi collects +extension load errors rather than aborting, so a failure is silent — run pi with +`-e` pointing at the entry point directly to isolate it. + +## Troubleshoot Missing LLM Spans + +There are none yet, and this is expected. pi's model traffic does not traverse +the gateway, so LLM lifecycle capture and model-call enforcement are both +unavailable. Tool and turn activity are unaffected. diff --git a/docs/reference/support-matrix.mdx b/docs/reference/support-matrix.mdx index e567b5c11..78cef3ab5 100644 --- a/docs/reference/support-matrix.mdx +++ b/docs/reference/support-matrix.mdx @@ -66,10 +66,12 @@ and older CLI versions during installation, diagnostics, and transparent runs. | --- | --- | --- | --- | | Claude Code | 2.1.121 | Persistent install, transparent run, lifecycle hooks, local gateway routing, and pre-tool security | Claude desktop, web, and application sessions are unsupported unless they expose the same local hook and gateway controls. Optimization requires gateway-routed LLM traffic and available hooks. | | Codex CLI | 0.143.0 | Persistent install, transparent run, 10 supported plugin hooks, local gateway routing, and pre-tool security | Cloud or remote tasks that bypass the local machine have partial or no LLM capture. The plugin hook schema has no `SessionEnd`; Relay finalizes the cumulative session snapshot at `Stop`. Encrypted Codex multi-agent v2 payloads cannot be decrypted or reliably linked. | +| pi | 0.84.0 | Transparent run through a Relay-authored pi extension, 11 lifecycle hooks, and tool-call security | Proof of concept. No persistent install: pi has no plugin marketplace. Model traffic does not traverse the gateway, so there are no LLM spans and no model-call enforcement. Subagents and nested pi processes appear as unrelated sessions. pi ships breaking changes through minor releases, so hook signatures need re-verification after an upgrade. | For installation, diagnostics, and host-specific behavior, refer to [Coding Agent Installation](/nemo-relay-cli/plugin-installation), [Claude -Code](/nemo-relay-cli/claude-code), and [Codex](/nemo-relay-cli/codex). +Code](/nemo-relay-cli/claude-code), [Codex](/nemo-relay-cli/codex), and +[pi](/nemo-relay-cli/pi). Hermes Agent includes NeMo Relay as a native in-process integration. It does not require a separate observability plugin or Relay CLI setup. Hermes Agent diff --git a/justfile b/justfile index abf785b90..85272e3d9 100644 --- a/justfile +++ b/justfile @@ -1141,6 +1141,7 @@ clean: integrations/openclaw/.test-dist \ integrations/openclaw/dist \ integrations/openclaw/node_modules \ + integrations/pi/node_modules \ node_modules \ docs/_build/ \ docs/reference/api/**/_generated/ \ @@ -1568,8 +1569,23 @@ test-openclaw: npm run test:live --workspace=nemo-relay-openclaw npm run pack:check --workspace=nemo-relay-openclaw +# --set [ci=true|false] +test-pi: + #!/usr/bin/env bash + {{ bash_helpers }} + cd "$NEMO_RELAY_REPO_ROOT" + if is_true "{{ ci }}"; then + npm ci --ignore-scripts + else + npm install --ignore-scripts + fi + # No `build-debug` for the Node binding: the pi extension is a sidecar HTTP + # client and loads no native addon, so nothing here depends on it. + npm run typecheck --workspace=nemo-relay-pi + npm test --workspace=nemo-relay-pi + # --set [output_dir=] [ci=true|false] -test-all: test-rust test-python test-python-langchain test-go test-node test-openclaw +test-all: test-rust test-python test-python-langchain test-go test-node test-openclaw test-pi # [version] or --set ref_name= set-version version="": diff --git a/package-lock.json b/package-lock.json index cd4f05d49..e571cfc3a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,8 @@ "name": "nemo-relay-workspace", "workspaces": [ "crates/node", - "integrations/openclaw" + "integrations/openclaw", + "integrations/pi" ], "devDependencies": { "fern-api": "5.57.0", @@ -476,6 +477,10 @@ "openclaw": "^2026.7.1" } }, + "integrations/pi": { + "name": "nemo-relay-pi", + "version": "0.8.0" + }, "node_modules/@boundaryml/baml": { "version": "0.219.0", "resolved": "https://registry.npmjs.org/@boundaryml/baml/-/baml-0.219.0.tgz", @@ -936,6 +941,10 @@ "resolved": "integrations/openclaw", "link": true }, + "node_modules/nemo-relay-pi": { + "resolved": "integrations/pi", + "link": true + }, "node_modules/openclaw": { "version": "2026.7.1", "resolved": "https://registry.npmjs.org/openclaw/-/openclaw-2026.7.1.tgz", diff --git a/package.json b/package.json index cf8803a04..0767b77a3 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ }, "workspaces": [ "crates/node", - "integrations/openclaw" + "integrations/openclaw", + "integrations/pi" ], "devDependencies": { "fern-api": "5.57.0", From ebe97d2079f494da719aad58389c274188bb6578 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 15:26:33 -0700 Subject: [PATCH 08/41] docs(pi): state the tool-result truncation and the subagent gap The two limitations left on the extension README's follow-through list, both of which a reader hits without warning. Tool results are cut at 2000 characters before forwarding, so the gateway records what a tool returned rather than necessarily all of it. And pi has no nested-agent hook, so subagents are not represented at all -- including the multi-process case, where a child pi process running this extension resolves its own session id and appears as an unrelated session rather than as a subagent. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- integrations/pi/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/integrations/pi/README.md b/integrations/pi/README.md index 6eb68cf78..a6485d79a 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -137,6 +137,23 @@ is recorded on the session scope rather than opening an empty turn to hold it. Codex and Claude Code report only `Stop`, so their turns stay lazily opened by the first event of the turn. +## What is not represented + +**Tool results are truncated at 2000 characters** before they are forwarded, with +the overflow replaced by a `... [truncated N chars]` suffix. The gateway +therefore records what a tool returned, not necessarily all of it — a large file +read or a long command output is cut. This keeps hook payloads bounded; raise +`MAX_CONTENT_CHARS` in `index.ts` if a policy needs to see more. + +**Subagents.** pi ships no nested-agent hook of its own — the extension has +nothing to derive a subagent id from — so `subagent_start` / `subagent_end` are +empty for pi and every tool span parents to the turn. The multi-process case is +worth stating separately: a child pi process running this extension resolves its +*own* session id and posts under it, so it does not appear as a subagent of the +parent. It appears as an unrelated session. + +**LLM spans**, until model redirection lands. See [Status](#status). + ## Development ```bash From d35d1b90593391666e21c9b6ef30bd0226696180 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 16:26:45 -0700 Subject: [PATCH 09/41] feat(pi): route pi model traffic through the gateway when it is safe Closes RELAY-732, the last real gap: pi's model calls now traverse the gateway, so LLM spans land in the same trace as tool and turn spans and a Relay guardrail can block a model call. The mechanism is one call, not a provider implementation. pi resolves a base URL per model from a generated catalog and has no base-URL flag or generic environment override, so redirection has to happen inside the extension. The ticket pointed at pi's `custom-provider-*` examples, which register a `streamSimple` and re-implement a provider protocol. That is the heavy path and it is not needed: `registerProvider(provider, { baseUrl })` with no `models` makes pi rewrite the URL of every existing model for that provider and keep their API, headers, costs and context windows (`applyExtension`, core/provider-composer.ts:215, verified in pi's source rather than taken from the doc comment). The extension stays a thin client. Redirection is conditional, and the condition is the design. The gateway forwards to one statically configured upstream per API family and a client cannot override it per request -- inbound internal dispatch headers are stripped, which is deliberate. So pointing a model at the gateway is only correct when the gateway's upstream is the endpoint that model would otherwise call. Redirecting an NVIDIA model into a gateway configured for api.openai.com does not degrade to "no spans"; it breaks a session that worked a moment earlier. The launcher therefore passes the gateway's own upstreams (NEMO_RELAY_PI_{OPENAI,ANTHROPIC}_UPSTREAM, from ResolvedConfig, which prepare_launch already had and pi ignored) and the extension redirects only on a match. Skips are recorded as a `model_redirect` mark naming the reason -- upstream-mismatch, unserviceable-api, unknown-upstream -- so a trace without LLM spans explains itself instead of looking broken. The decision is re-made on every model_select. NEMO_RELAY_PI_REDIRECT=force skips the check, =off disables it. Turn boundaries now block, because model traffic does not use the hook queue. Reading the first redirected trace found a real defect: an LLM span opened under the previous turn. pi sends model requests to the gateway directly over HTTP while observability hooks go through the extension's serial queue, so the next turn's model request beat our queued `turn_end` and was parented by the turn that was still open. `turn_start` and `turn_end` are now awaited. Two local round trips per turn buys correct parenting, on the same reasoning that already makes tool_call await -- a span opened under the wrong turn is simply wrong. The re-captured trace has every span closing inside the scope that opened it. Verified against live pi v0.84.0 with a real model: three LLM spans nested under their own turns alongside the tool span, and separately, with the example policy plugin configured block_llms = true, a real guardrail rejecting a real pi model call -- pi surfaced it as a clean 403 and the trace recorded the rejection as a mark rather than a span, because the call never executed. Also corrects two counts the ticket carried: pi ships 38 providers, not 39, and 6 of them speak an API the gateway has no route for, not 7 -- "Radius" is an OAuth mode, not a provider. Green: 1164 + 12 + 102 Rust, 42 Node, tsc clean, docs-linkcheck 0 errors, clippy -D warnings clean, pre-commit clean apart from cargo-deny/gofmt/go-vet, which are not installed on this machine. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- .gitignore | 3 + README.md | 2 +- crates/cli/src/agents/mod.rs | 7 +- crates/cli/src/agents/pi/launch.rs | 49 +++- crates/cli/src/agents/pi/mod.rs | 3 + .../coverage/agents/coding_agent_tests.rs | 2 +- .../tests/coverage/agents/launcher_tests.rs | 63 +++++ docs/nemo-relay-cli/about.mdx | 2 +- docs/nemo-relay-cli/basic-usage.mdx | 2 +- docs/nemo-relay-cli/pi.mdx | 79 ++++++- docs/reference/support-matrix.mdx | 2 +- integrations/pi/README.md | 55 ++++- integrations/pi/index.ts | 90 ++++++- integrations/pi/src/pi-hook-types.ts | 30 +++ integrations/pi/src/provider-redirect.ts | 222 ++++++++++++++++++ .../pi/test/provider-redirect.test.mjs | 169 +++++++++++++ 16 files changed, 744 insertions(+), 36 deletions(-) create mode 100644 integrations/pi/src/provider-redirect.ts create mode 100644 integrations/pi/test/provider-redirect.test.mjs diff --git a/.gitignore b/.gitignore index 61814f76c..2fa985d81 100644 --- a/.gitignore +++ b/.gitignore @@ -86,3 +86,6 @@ CHANGELOG.md # Relay /.nemo-relay /artifacts/ + +# Generated per machine by the dynamic-plugin recipe: local dylib name + sha256. +examples/**/relay-plugin.local.toml diff --git a/README.md b/README.md index 9ba09612d..8483bc678 100644 --- a/README.md +++ b/README.md @@ -315,7 +315,7 @@ coverage. |:--|:--:|:--:|:--:|:--| | Claude Code | Yes | Yes | Partial | Hook forwarding, pre-tool blocking, and gateway-routed LLM observability are supported. | | Codex | Yes | Yes | Partial | Persistent install verifies the exact plugin hooks. Each `Stop` finalizes a turn snapshot; the supported generated schema does not install `SessionEnd`. | -| pi | Partial | Partial | No | Proof of concept. A Relay-authored pi extension forwards tool and turn activity and gates tool calls; pi's model traffic does not traverse the gateway, so there are no LLM spans. | +| pi | Partial | Yes | No | Proof of concept. A Relay-authored pi extension forwards tool and turn activity, gates tool calls, and points the active model's provider at the gateway when the gateway fronts that provider — which is what enables model-call enforcement and LLM spans. | | Hermes Agent | Yes | Yes | Partial | NeMo Relay is built into Hermes Agent, and Hermes Agent understands NeMo Relay plugin configurations. No separate observability plugin or Relay CLI setup is required. | ### Public API Integrations diff --git a/crates/cli/src/agents/mod.rs b/crates/cli/src/agents/mod.rs index 9cdfae8ce..5dcb7b9a6 100644 --- a/crates/cli/src/agents/mod.rs +++ b/crates/cli/src/agents/mod.rs @@ -398,7 +398,7 @@ pub(crate) fn prepare_launch( agent: CodingAgent, launch: &mut crate::process::PreparedAgentLaunch, gateway_url: &str, - _resolved: &crate::configuration::ResolvedConfig, + resolved: &crate::configuration::ResolvedConfig, proxy_credential: &crate::provider_auth::TransparentProxyCredential, dry_run: bool, ) -> Result<(), crate::error::CliError> { @@ -411,7 +411,10 @@ pub(crate) fn prepare_launch( CodingAgent::ClaudeCode => { claude::launch::prepare(launch, gateway_url, proxy_credential, dry_run) } - CodingAgent::Pi => pi::launch::prepare(launch, gateway_url), + // pi is the only agent whose launcher needs the gateway's upstream configuration: its + // extension redirects model traffic only when the selected model already targets that + // upstream. See `pi::launch::prepare`. + CodingAgent::Pi => pi::launch::prepare(launch, gateway_url, &resolved.gateway), } } diff --git a/crates/cli/src/agents/pi/launch.rs b/crates/cli/src/agents/pi/launch.rs index dde07002f..f272c8945 100644 --- a/crates/cli/src/agents/pi/launch.rs +++ b/crates/cli/src/agents/pi/launch.rs @@ -29,9 +29,34 @@ pub(crate) const PI_GATEWAY_URL_ENV: &str = "NEMO_RELAY_PI_GATEWAY_URL"; /// Environment variable pointing pi at the NeMo Relay extension entry point. pub(crate) const PI_EXTENSION_PATH_ENV: &str = "NEMO_RELAY_PI_EXTENSION"; -pub(crate) fn prepare(launch: &mut PreparedAgentLaunch, gateway_url: &str) -> Result<(), CliError> { +/// Upstream this gateway forwards OpenAI-compatible traffic to. +pub(crate) const PI_OPENAI_UPSTREAM_ENV: &str = "NEMO_RELAY_PI_OPENAI_UPSTREAM"; + +/// Upstream this gateway forwards Anthropic traffic to. +pub(crate) const PI_ANTHROPIC_UPSTREAM_ENV: &str = "NEMO_RELAY_PI_ANTHROPIC_UPSTREAM"; + +pub(crate) fn prepare( + launch: &mut PreparedAgentLaunch, + gateway_url: &str, + gateway: &crate::configuration::GatewayConfig, +) -> Result<(), CliError> { set_env(launch, PI_GATEWAY_URL_ENV, gateway_url); + // Tell the extension what this gateway actually forwards to. + // + // Redirection is only correct when the gateway's upstream is the same endpoint the selected + // model would otherwise call: the gateway resolves one OpenAI base and one Anthropic base from + // static configuration (`ProviderRoute::upstream_url`) and there is no per-request override a + // client can set -- inbound internal dispatch headers are stripped. Without these two values + // the extension would have to redirect blind, and pointing (say) an NVIDIA model at a gateway + // configured for `api.openai.com` breaks a session that worked a moment earlier. + set_env(launch, PI_OPENAI_UPSTREAM_ENV, &gateway.openai_base_url); + set_env( + launch, + PI_ANTHROPIC_UPSTREAM_ENV, + &gateway.anthropic_base_url, + ); + // `-e` is the right loader here: it is trust-ungated, loads before // discovery, and survives `--no-extensions`, so a launched session gets the // extension regardless of the user's own pi configuration. @@ -49,16 +74,18 @@ pub(crate) fn prepare(launch: &mut PreparedAgentLaunch, gateway_url: &str) -> Re ["-e".to_string(), rendered], ); - // Do not claim redirection here: the extension does not yet register a - // gateway-backed provider, so model calls still go straight to the - // provider. Only tool and turn activity reaches Relay today. - launch.notes.push( - "pi tool and turn activity is reported to NeMo Relay by the extension; model calls are \ - NOT yet routed through the gateway, so there are no LLM spans. pi has no base-URL flag \ - or generic environment override, so redirection requires the extension to register a \ - gateway-backed provider" - .to_string(), - ); + // Redirection is conditional, so say what the condition is rather than promising LLM spans. + launch.notes.push(format!( + "pi tool and turn activity is reported to NeMo Relay by the extension. Model calls are \ + routed through the gateway only when the selected model's provider already targets this \ + gateway's upstream (openai={openai}, anthropic={anthropic}); pi resolves a base URL per \ + model from a generated catalog, and the gateway forwards to one statically configured \ + upstream per API family. A model on any other provider keeps calling its own endpoint \ + and produces no LLM spans -- select a matching model, or start the gateway with \ + --openai-base-url / --anthropic-base-url pointing at that provider", + openai = gateway.openai_base_url, + anthropic = gateway.anthropic_base_url, + )); Ok(()) } diff --git a/crates/cli/src/agents/pi/mod.rs b/crates/cli/src/agents/pi/mod.rs index 02d5fadf3..f0785ca0b 100644 --- a/crates/cli/src/agents/pi/mod.rs +++ b/crates/cli/src/agents/pi/mod.rs @@ -44,6 +44,9 @@ pub(super) const DESCRIPTOR: AgentDescriptor = AgentDescriptor { "turn_end", "tool_call", "tool_execution_end", + // Not a pi hook name: the extension posts this after deciding whether to point the active + // model's provider at the gateway, so a trace with no LLM spans carries its own reason. + "model_redirect", ], }; diff --git a/crates/cli/tests/coverage/agents/coding_agent_tests.rs b/crates/cli/tests/coverage/agents/coding_agent_tests.rs index b27b1521f..71698eba3 100644 --- a/crates/cli/tests/coverage/agents/coding_agent_tests.rs +++ b/crates/cli/tests/coverage/agents/coding_agent_tests.rs @@ -22,7 +22,7 @@ fn agent_descriptors_are_complete_and_unique() { assert_eq!(CodingAgent::ClaudeCode.hook_events().len(), 14); assert_eq!(CodingAgent::Codex.hook_events().len(), 10); assert_eq!(CodingAgent::Pi.label(), "pi"); - assert_eq!(CodingAgent::Pi.hook_events().len(), 11); + assert_eq!(CodingAgent::Pi.hook_events().len(), 12); for agent in CodingAgent::ALL { let events = agent.hook_events(); assert!(events.iter().all(|event| !event.is_empty())); diff --git a/crates/cli/tests/coverage/agents/launcher_tests.rs b/crates/cli/tests/coverage/agents/launcher_tests.rs index d747cb8ca..6deb4e2d5 100644 --- a/crates/cli/tests/coverage/agents/launcher_tests.rs +++ b/crates/cli/tests/coverage/agents/launcher_tests.rs @@ -1640,3 +1640,66 @@ fn make_executable(path: &Path) { permissions.set_mode(0o755); std::fs::set_permissions(path, permissions).unwrap(); } + +// The extension redirects pi's model traffic only when the gateway forwards to the endpoint the +// selected model would otherwise call. It cannot discover that on its own -- the gateway resolves +// one upstream per API family from static configuration and strips client-supplied dispatch +// headers -- so the launcher has to tell it. Without these two variables the extension has no safe +// basis to redirect and deliberately stays put, which shows up as a session with no LLM spans. +#[test] +fn pi_launch_passes_the_gateway_upstreams_the_extension_redirects_against() { + let _guard = current_dir_lock().lock().unwrap(); + let extension = std::env::temp_dir().join("nemo-relay-pi-launch-test-extension.ts"); + std::fs::write(&extension, "export default () => {};").unwrap(); + let _env = EnvScope::set(&[( + crate::agents::pi::launch::PI_EXTENSION_PATH_ENV, + Some(extension.as_os_str()), + )]); + + let resolved = ResolvedConfig { + gateway: GatewayConfig { + openai_base_url: "https://integrate.api.nvidia.com/v1".into(), + anthropic_base_url: "https://anthropic.internal.example/".into(), + ..GatewayConfig::default() + }, + agents: AgentConfigs::default(), + ..ResolvedConfig::default() + }; + let prepared = PreparedAgentLaunch::new( + CodingAgent::Pi, + vec!["pi".into()], + "http://127.0.0.1:4040", + &resolved, + false, + ) + .unwrap(); + + let env = |name: &str| { + prepared + .env + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.as_str()) + }; + assert_eq!( + env(crate::agents::pi::launch::PI_OPENAI_UPSTREAM_ENV), + Some("https://integrate.api.nvidia.com/v1"), + ); + assert_eq!( + env(crate::agents::pi::launch::PI_ANTHROPIC_UPSTREAM_ENV), + Some("https://anthropic.internal.example/"), + ); + assert_eq!( + env(crate::agents::pi::launch::PI_GATEWAY_URL_ENV), + Some("http://127.0.0.1:4040"), + ); + + // The note has to state the condition, not promise LLM spans: a model on any other provider + // keeps calling its own endpoint. + let note = prepared.notes.join(" "); + assert!( + note.contains("https://integrate.api.nvidia.com/v1"), + "the launch note should name the upstream redirection is judged against: {note}" + ); + let _ = std::fs::remove_file(&extension); +} diff --git a/docs/nemo-relay-cli/about.mdx b/docs/nemo-relay-cli/about.mdx index 5378627b8..646c21a7b 100644 --- a/docs/nemo-relay-cli/about.mdx +++ b/docs/nemo-relay-cli/about.mdx @@ -58,7 +58,7 @@ controls. | --- | --- | --- | --- | --- | | Claude Code | Yes | Yes | Partial | Pre-tool hook responses are supported. LLM optimization uses gateway-routed traffic; full coverage depends on loaded Claude Code hooks. | | Codex | Yes | Yes | Partial | Persistent install verifies all 10 hooks in the supported schema. Each `Stop` finalizes a turn snapshot because the plugin schema does not expose `SessionEnd`. | -| pi | Partial | Partial | No | Proof of concept. Tool and turn activity are captured through a Relay-authored pi extension, and tool calls can be blocked. pi's model traffic does not traverse the gateway, so there are no LLM spans and no model-call enforcement. | +| pi | Partial | Yes | No | Proof of concept. Tool and turn activity are captured through a Relay-authored pi extension, and both tool calls and model calls can be blocked. Model traffic reaches the gateway only when the gateway forwards to the endpoint the selected model would otherwise call. | For minimum agent versions, platform support, and current limitations, refer to the [Support Matrix](/reference/support-matrix). diff --git a/docs/nemo-relay-cli/basic-usage.mdx b/docs/nemo-relay-cli/basic-usage.mdx index 8f0b43267..b68ddccc3 100644 --- a/docs/nemo-relay-cli/basic-usage.mdx +++ b/docs/nemo-relay-cli/basic-usage.mdx @@ -523,7 +523,7 @@ Generated hook bundles subscribe to the events needed for that mapping: | --- | --- | --- | | Claude Code | `UserPromptSubmit`, `Stop` | `SessionStart`, `SessionEnd`, `UserPromptExpansion`, `SubagentStart`, `SubagentStop`, `PreToolUse`, `PostToolUse`, `PostToolUseFailure`, `PermissionRequest`, `Notification`, `PreCompact`, `PostCompact` | | Codex | `UserPromptSubmit`, `Stop` | `SessionStart`, `SubagentStart`, `SubagentStop`, `PreToolUse`, `PostToolUse`, `PermissionRequest`, `PreCompact`, `PostCompact` | -| pi | none yet — pi's model traffic does not reach the gateway | `session_start`, `session_shutdown`, `agent_start`, `agent_end`, `agent_settled`, `turn_start`, `turn_end`, `session_before_compact`, `session_compact`, `tool_call`, `tool_execution_end` | +| pi | none — pi's LLM spans come from real gateway traffic, not from hooks | `session_start`, `session_shutdown`, `agent_start`, `agent_end`, `agent_settled`, `turn_start`, `turn_end`, `session_before_compact`, `session_compact`, `tool_call`, `tool_execution_end` | ## Hook Forwarding diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index 1b062f4d8..8402f780c 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -8,9 +8,11 @@ SPDX-License-Identifier: Apache-2.0 */} Use this guide to observe local [pi](https://github.com/earendil-works/pi) -sessions and to gate pi tool calls on a NeMo Relay policy. The pi integration is -a proof of concept: tool and turn activity reach Relay, and LLM traffic does not -yet, so there are no LLM spans and no model-call enforcement. +sessions and to gate pi tool calls and model calls on a NeMo Relay policy. The pi +integration is a proof of concept. Tool and turn activity always reach Relay; +model traffic reaches it when the gateway forwards to the endpoint the selected +model would otherwise call, which is what [Model +Redirection](#model-redirection) explains. pi ships breaking changes through *minor* releases and has no major-release @@ -39,10 +41,11 @@ Three consequences follow from that shape: handlers on its critical path, so a blocking hook is synchronous by construction. Observability-only hooks are queued rather than awaited and drained at shutdown, so they do not charge that path. -- Model traffic is not redirected. pi resolves `baseUrl` per model from a - generated catalog and has no base-URL flag or generic environment override, - so redirection requires the extension to register a gateway-backed provider. - That work is not done yet. +- Model traffic is redirected by the extension, not by configuration. pi + resolves `baseUrl` per model from a generated catalog and has no base-URL flag + or generic environment override, so the extension calls + `registerProvider(provider, { baseUrl })` itself — and only when doing so is + safe. See [Model Redirection](#model-redirection). ## Requirements @@ -94,6 +97,9 @@ NEMO_RELAY_PI_GATEWAY_URL=http://127.0.0.1:4040 \ | `NEMO_RELAY_PI_GATEWAY_URL` | `http://127.0.0.1:4040` | Gateway base URL | | `NEMO_RELAY_PI_TIMEOUT_MS` | `5000` | Per-request timeout | | `NEMO_RELAY_PI_FAIL` | `open` | Set to `closed` to block tool calls when the gateway is unreachable | +| `NEMO_RELAY_PI_REDIRECT` | `match` | `force` redirects without verifying the upstream; `off` disables redirection | +| `NEMO_RELAY_PI_OPENAI_UPSTREAM` | unset | What the gateway forwards OpenAI-compatible traffic to. Set by `nemo-relay run` | +| `NEMO_RELAY_PI_ANTHROPIC_UPSTREAM` | unset | What the gateway forwards Anthropic traffic to. Set by `nemo-relay run` | Run headless pi with `< /dev/null`. pi drains piped standard input before the @@ -143,6 +149,7 @@ empty turn to hold it. | `turn_start` / `turn_end` | Turn scope open and close | | `session_before_compact` | Mark. The compaction is announced, not yet done, and a later extension can still cancel it | | `session_compact` | Canonical `compaction` mark | +| `model_select` | `model_redirect` mark recording whether redirection applied | | `tool_call` | Tool span start, and the gate | | `tool_execution_end` | Tool span end, for every outcome including blocked | @@ -156,6 +163,53 @@ before the span opens, but pi fires `tool_execution_end` for blocked calls too, and the gateway synthesizes the missing pair, tagged `tool_correlation_status: "agent_fallback"`. +## Model Redirection + +pi has no base-URL flag and no generic environment override — it resolves a base +URL per model from a generated catalog — so the extension points the active +model's provider at the gateway directly, with +`registerProvider(provider, { baseUrl })`. pi rewrites the URL of every existing +model for that provider and keeps their API, headers, costs and context windows. + +Redirection is **conditional**. The gateway forwards to one statically +configured upstream per API family, set by `--openai-base-url` and +`--anthropic-base-url`, and a client cannot override that per request. +Redirecting is therefore only correct when the gateway's upstream is the endpoint +the selected model would otherwise have called. Pointing an NVIDIA model at a +gateway configured for `api.openai.com` does not produce a trace without spans; +it breaks the session. + +`nemo-relay run --agent pi` passes the gateway's upstreams to the extension, and +the extension redirects only on a match: + +| Situation | Outcome | +| --- | --- | +| Gateway upstream equals the model's endpoint | Redirected; LLM spans appear under the turn | +| Gateway forwards elsewhere | Skipped (`upstream-mismatch`) | +| Model's API has no gateway route | Skipped (`unserviceable-api`) | +| Upstream unknown, e.g. a standalone gateway | Skipped (`unknown-upstream`) | + +Every outcome is recorded as a `model_redirect` mark on the session scope, so a +trace with no LLM spans states its own reason. The decision is re-made on each +model switch. + +To capture a specific provider, point the gateway at it: + +```bash +nemo-relay --bind 127.0.0.1:4040 \ + --openai-base-url https://integrate.api.nvidia.com/v1 +``` + +32 of pi's 38 providers speak an API the gateway serves. The six that do not are +Amazon Bedrock, Azure OpenAI Responses, Google, Google Vertex, Mistral, and +OpenAI Codex. + + +Point pi at the gateway **root**, not the root plus `/v1`. The Anthropic SDK +appends `/v1/messages` itself, while the OpenAI SDK appends `/chat/completions`; +the gateway serves both shapes from the root. + + ## Agent-Run Attribution One pi prompt can re-enter the agent run several times — provider retry, @@ -211,6 +265,11 @@ extension load errors rather than aborting, so a failure is silent — run pi wi ## Troubleshoot Missing LLM Spans -There are none yet, and this is expected. pi's model traffic does not traverse -the gateway, so LLM lifecycle capture and model-call enforcement are both -unavailable. Tool and turn activity are unaffected. +Look for the `model_redirect` mark on the session scope: it names the outcome and +the reason. The common ones are `upstream-mismatch` (start the gateway with +`--openai-base-url` or `--anthropic-base-url` pointing at the model's provider), +`unknown-upstream` (launch through `nemo-relay run --agent pi`, or set +`NEMO_RELAY_PI_REDIRECT=force`), and `unserviceable-api` (the model's provider +speaks an API the gateway has no route for — pick another model). + +Tool and turn activity are unaffected by any of these. diff --git a/docs/reference/support-matrix.mdx b/docs/reference/support-matrix.mdx index 78cef3ab5..d61821b3a 100644 --- a/docs/reference/support-matrix.mdx +++ b/docs/reference/support-matrix.mdx @@ -66,7 +66,7 @@ and older CLI versions during installation, diagnostics, and transparent runs. | --- | --- | --- | --- | | Claude Code | 2.1.121 | Persistent install, transparent run, lifecycle hooks, local gateway routing, and pre-tool security | Claude desktop, web, and application sessions are unsupported unless they expose the same local hook and gateway controls. Optimization requires gateway-routed LLM traffic and available hooks. | | Codex CLI | 0.143.0 | Persistent install, transparent run, 10 supported plugin hooks, local gateway routing, and pre-tool security | Cloud or remote tasks that bypass the local machine have partial or no LLM capture. The plugin hook schema has no `SessionEnd`; Relay finalizes the cumulative session snapshot at `Stop`. Encrypted Codex multi-agent v2 payloads cannot be decrypted or reliably linked. | -| pi | 0.84.0 | Transparent run through a Relay-authored pi extension, 11 lifecycle hooks, and tool-call security | Proof of concept. No persistent install: pi has no plugin marketplace. Model traffic does not traverse the gateway, so there are no LLM spans and no model-call enforcement. Subagents and nested pi processes appear as unrelated sessions. pi ships breaking changes through minor releases, so hook signatures need re-verification after an upgrade. | +| pi | 0.84.0 | Transparent run through a Relay-authored pi extension, 11 lifecycle hooks, tool-call security, and model-call security when redirection applies | Proof of concept. No persistent install: pi has no plugin marketplace. Model traffic is redirected only when the gateway forwards to the endpoint the selected model would otherwise call; otherwise there are no LLM spans for that model. Six of pi's 38 providers speak an API the gateway has no route for. Subagents and nested pi processes appear as unrelated sessions. pi ships breaking changes through minor releases, so hook signatures need re-verification after an upgrade. | For installation, diagnostics, and host-specific behavior, refer to [Coding Agent Installation](/nemo-relay-cli/plugin-installation), [Claude diff --git a/integrations/pi/README.md b/integrations/pi/README.md index a6485d79a..da37bb7f0 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -24,12 +24,13 @@ Proof of concept, tracked under `v0.84.0`. pi ships breaking changes through *minor* releases and has no major-release channel, so re-verify hook signatures before relying on them. -**Model traffic does not traverse the gateway yet** +**Model traffic is redirected conditionally** ([RELAY-732](https://linear.app/nvidia/issue/RELAY-732)). pi has no base-URL flag and no generic environment override — it resolves `baseUrl` per model from -a generated catalog — so redirection needs this extension to register a -gateway-backed provider, and nothing does that yet. Until it lands there are no -LLM spans and no model-call enforcement, only tool and turn activity. +a generated catalog — so the extension points the active model's provider at the +gateway itself. It only does so when the gateway forwards to the endpoint that +model would otherwise have called; see [Model redirection](#model-redirection). +When it does not, you get tool and turn activity but no LLM spans. ## Usage @@ -53,6 +54,9 @@ directory (`~/.pi/agent/extensions/`, `.pi/extensions/`). | `NEMO_RELAY_PI_GATEWAY_URL` | `http://127.0.0.1:4040` | Gateway base URL | | `NEMO_RELAY_PI_TIMEOUT_MS` | `5000` | Per-request timeout | | `NEMO_RELAY_PI_FAIL` | `open` | `closed` blocks tool calls when the gateway is unreachable | +| `NEMO_RELAY_PI_REDIRECT` | `match` | `force` redirects without checking the upstream; `off` disables redirection | +| `NEMO_RELAY_PI_OPENAI_UPSTREAM` | unset | What the gateway forwards OpenAI-compatible traffic to. Set by the launcher | +| `NEMO_RELAY_PI_ANTHROPIC_UPSTREAM` | unset | What the gateway forwards Anthropic traffic to. Set by the launcher | ## How tool gating works @@ -75,6 +79,44 @@ The block reason reaches the model **verbatim**: pi hands it to as error codes — a reason that says what to do instead produces a model that adapts rather than one that gives up. +## Model redirection + +pi resolves a base URL per model from a generated catalog, so there is no flag or +environment variable to point it at the gateway. The extension does it directly: + +```ts +pi.registerProvider(providerId, { baseUrl: gatewayUrl }); +``` + +With `baseUrl` and no `models`, pi rewrites the URL of every existing model for +that provider and keeps their API, headers, costs and context windows. That is +much cheaper than pi's own `custom-provider-*` examples, which register a +`streamSimple` and re-implement a provider protocol. + +**It is conditional, and the condition is the point.** The gateway forwards to +one statically configured upstream per API family — `--openai-base-url` and +`--anthropic-base-url` — and a client cannot override that per request. So +redirecting is only correct when the gateway's upstream *is* the endpoint the +selected model would otherwise call. Pointing an NVIDIA model at a gateway +configured for `api.openai.com` does not degrade to "no spans"; it breaks the +session. The extension therefore redirects only on a match, and records every +outcome as a `model_redirect` mark so a trace without LLM spans explains itself. + +| Situation | Outcome | +|---|---| +| Gateway upstream equals the model's endpoint | Redirected; LLM spans appear under the turn | +| Gateway forwards somewhere else | Skipped, `upstream-mismatch` | +| Model's API has no gateway route (Bedrock, Azure OpenAI Responses, Google, Google Vertex, Mistral, OpenAI Codex) | Skipped, `unserviceable-api` | +| Launched outside `nemo-relay run --agent pi`, so the upstream is unknown | Skipped, `unknown-upstream` — set `NEMO_RELAY_PI_REDIRECT=force` to override | + +`nemo-relay run --agent pi` sets the two upstream variables for you. Running pi +by hand against a standalone gateway means setting them yourself, or forcing. + +The decision is re-evaluated on every `model_select`, so switching to a model the +gateway does not front stops redirecting rather than silently misrouting. + +32 of pi's 38 providers speak an API the gateway serves; the six above do not. + ## Hook mapping pi's lifecycle is `session -> agent run -> turn -> message | tool execution`. @@ -120,8 +162,9 @@ per-call state is keyed by `toolCallId`, the only correlator pi provides. | `session_start` / `session_shutdown` | session boundary | **Not** `agent_start`/`agent_end` — those repeat on re-entry. `session_shutdown` is ignored for `reason: "reload"`, which continues the same session | | `agent_start` / `agent_end` | run-level marks | Carry `attempt_index`; not a run boundary. Recorded on the session scope, not inside a turn | | `agent_settled` | run-level mark | Fires exactly once, from a `finally`. Carries `attempts` (the count) and `attempt_index` (the last one) | -| `turn_start` | turn scope **open** | Carries `turn_index`, `turn_seq`, `attempt_index` | -| `turn_end` | turn scope **close** | Carries `turn_index`, `turn_seq`, `attempt_index` | +| `turn_start` | turn scope **open** | Carries `turn_index`, `turn_seq`, `attempt_index`. Awaited, so the turn exists before pi's model call arrives | +| `turn_end` | turn scope **close** | Carries `turn_index`, `turn_seq`, `attempt_index`. Awaited, for the same reason | +| `model_select` | `model_redirect` mark | Re-evaluates redirection for the newly selected model | | `session_before_compact` | mark | Announced, not done, and cancellable by a later extension. Carries `reason`, `will_retry`, `tokens_before` | | `session_compact` | compaction | The completed compaction, which the runtime treats as proof the context was rebuilt | | `tool_call` | tool start, and the gate | The only blocking hook. Carries `attempt_index`, `turn_seq` | diff --git a/integrations/pi/index.ts b/integrations/pi/index.ts index 71d3d29b0..44ea5b2a3 100644 --- a/integrations/pi/index.ts +++ b/integrations/pi/index.ts @@ -38,10 +38,17 @@ * * Load it with `pi -e `, or let `nemo-relay launch pi` do it. * + * **Model redirection.** pi resolves a base URL per model from a generated + * catalog, so the extension points the active model's provider at the gateway + * itself -- but only when the gateway forwards to the endpoint that model would + * otherwise call. See `src/provider-redirect.ts`. + * * Environment (set by the launcher, overridable by hand): * - `NEMO_RELAY_PI_GATEWAY_URL` gateway base URL (default `http://127.0.0.1:4040`) * - `NEMO_RELAY_PI_TIMEOUT_MS` per-request timeout (default 5000) * - `NEMO_RELAY_PI_FAIL` `closed` to block when the gateway is unreachable + * - `NEMO_RELAY_PI_REDIRECT` `force` to skip the upstream check, `off` to disable + * - `NEMO_RELAY_PI_{OPENAI,ANTHROPIC}_UPSTREAM` what the gateway forwards to */ import { type GatewayConfig, @@ -50,12 +57,19 @@ import { postHook, resolveFault, } from './src/gateway-client.ts'; +import { + type RedirectConfig, + decideRedirect, + isNotable, + redirectConfigFromEnv, +} from './src/provider-redirect.ts'; import type { AgentEndEvent, AgentSettledEvent, AgentStartEvent, ExtensionAPI, ExtensionContext, + ModelSelectEvent, SessionBeforeCompactEvent, SessionCompactEvent, SessionShutdownEvent, @@ -126,6 +140,27 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { void enqueue(() => postAndForget(active, payload)); }; + /** + * Forward a hook and wait for the gateway to have processed it. + * + * Used only for the two turn boundaries. Everything else the extension sends + * is observability that can settle late, but a turn boundary *defines the + * parent* of whatever comes next -- and model traffic does not travel through + * this queue at all. pi sends model requests to the gateway directly over + * HTTP, so a queued `turn_end` races them: an acceptance trace showed the + * next turn's LLM span opening under the previous turn, because pi's request + * beat our post. Awaiting these two costs two local round trips per turn and + * removes the race, on the same reasoning that already makes `tool_call` + * await -- a span opened under the wrong turn is simply wrong. + */ + const emitOrdered = async ( + ctx: ExtensionContext, + payload: Record, + ): Promise => { + const active = ensureConfig(ctx); + await enqueue(() => postAndForget(active, payload)); + }; + /** * The attempt and turn a hook belongs to. * @@ -146,6 +181,46 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { turn_seq: Math.max(0, turnSeq - 1), }); + /** Providers already pointed at the gateway, so the redirect is idempotent. */ + const redirectedProviders = new Set(); + let redirect: RedirectConfig | null = null; + + /** + * Point the active model's provider at the gateway, when that is safe. + * + * Runs on `session_start` and again on every `model_select`, because the + * decision is per-model: switching from a provider the gateway fronts to one + * it does not must not leave the new provider redirected. Each outcome is + * reported to the gateway as a mark, so a trace with no LLM spans says why in + * the trace itself rather than only in the user's terminal. + */ + const applyRedirect = (ctx: ExtensionContext, source: string): void => { + const active = ensureConfig(ctx); + redirect ??= redirectConfigFromEnv(active.url); + const decision = decideRedirect(ctx.model, redirect, redirectedProviders); + if (decision.kind === 'redirect') { + // Only baseUrl: pi rewrites the URL of every existing model for this + // provider and keeps their API, headers and costs. + pi.registerProvider(decision.provider, { baseUrl: redirect.gatewayUrl }); + redirectedProviders.add(decision.provider); + } + // A transient skip -- no model resolved yet, or a provider already pointed + // at the gateway -- explains nothing, and a mark per session_start for it + // is noise in every trace. + if (!isNotable(decision)) return; + emit(ctx, { + hook_event_name: 'model_redirect', + source, + outcome: decision.kind, + reason: decision.reason, + ...(decision.provider ? { provider: decision.provider } : {}), + ...(decision.api ? { model_api: decision.api } : {}), + ...(decision.kind === 'redirect' ? { upstream: decision.upstream } : {}), + ...(ctx.model ? { model_id: ctx.model.id } : {}), + ...attribution(), + }); + }; + // --------------------------------------------------------------------------- // Session lifecycle // @@ -156,6 +231,17 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { pi.on('session_start', async (event: SessionStartEvent, ctx: ExtensionContext) => { emit(ctx, { hook_event_name: 'session_start', reason: event.reason, cwd: ctx.cwd }); + applyRedirect(ctx, 'session_start'); + }); + + /** + * Re-evaluate on every model switch. + * + * pi fires this for the initial selection too, so a session that resolves its + * model after `session_start` is still covered. + */ + pi.on('model_select', async (_event: ModelSelectEvent, ctx: ExtensionContext) => { + applyRedirect(ctx, 'model_select'); }); /** @@ -224,7 +310,7 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { pi.on('turn_start', async (event: TurnStartEvent, ctx: ExtensionContext) => { const seq = turnSeq; turnSeq += 1; - emit(ctx, { + await emitOrdered(ctx, { hook_event_name: 'turn_start', // pi's turn_index resets to 0 on re-entry; turn_seq does not, so a // consumer can still order turns across the whole session. @@ -235,7 +321,7 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { }); pi.on('turn_end', async (event: TurnEndEvent, ctx: ExtensionContext) => { - emit(ctx, { + await emitOrdered(ctx, { hook_event_name: 'turn_end', // pi carries turn_index on the close but not turn_seq, so the close could // not be matched to its own open across a re-entry, where turn_index 0 diff --git a/integrations/pi/src/pi-hook-types.ts b/integrations/pi/src/pi-hook-types.ts index a5b130637..c47019e8d 100644 --- a/integrations/pi/src/pi-hook-types.ts +++ b/integrations/pi/src/pi-hook-types.ts @@ -131,12 +131,30 @@ export type ToolCallEventResult = { reason?: string; }; +/** A model, narrowed to the fields this extension reads. */ +export type PiModel = { + id: string; + api: string; + provider: string; + baseUrl: string; +}; + +/** Fired when a model is selected, including the initial selection. */ +export type ModelSelectEvent = { + type: 'model_select'; + model: PiModel; + previousModel?: PiModel; + source?: string; +}; + /** Minimal view of pi's extension context. */ export type ExtensionContext = { cwd: string; mode: string; hasUI: boolean; sessionManager: { getSessionId(): string }; + /** The active model. Undefined before one is resolved. */ + model?: PiModel; }; export type ExtensionHandler = ( @@ -158,4 +176,16 @@ export type ExtensionAPI = { on(event: 'tool_execution_start', handler: ExtensionHandler): void; on(event: 'tool_execution_end', handler: ExtensionHandler): void; on(event: 'tool_call', handler: ExtensionHandler): void; + on(event: 'model_select', handler: ExtensionHandler): void; + + /** + * Register or override a model provider. + * + * With `baseUrl` and no `models`, pi rewrites the URL of every existing model + * for that provider and preserves their API, headers and costs + * (`core/provider-composer.ts:215`). During initial extension load the call is + * queued and applied once the runner binds its context, so calling it from a + * factory is safe. + */ + registerProvider(name: string, config: { baseUrl?: string }): void; }; diff --git a/integrations/pi/src/provider-redirect.ts b/integrations/pi/src/provider-redirect.ts new file mode 100644 index 000000000..ef9307f4e --- /dev/null +++ b/integrations/pi/src/provider-redirect.ts @@ -0,0 +1,222 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Points pi's model traffic at the NeMo Relay gateway. + * + * pi has no base-URL flag and no generic environment override -- it resolves a + * base URL per model from a generated catalog -- so redirection has to happen + * inside the extension. The mechanism is one call: + * + * ```ts + * pi.registerProvider(providerId, { baseUrl: gatewayUrl }); + * ``` + * + * With `baseUrl` and no `models`, pi rewrites the URL of every existing model + * for that provider and keeps their API, headers, costs and context windows + * (`applyExtension`, `core/provider-composer.ts:215`). That is far cheaper than + * pi's own `custom-provider-*` examples, which register a `streamSimple` and + * re-implement a provider protocol; the extension stays a thin client. + * + * **Redirection is conditional, and the condition is the whole design.** The + * gateway forwards to one statically configured upstream per API family and a + * client cannot override it per request -- inbound internal dispatch headers + * are stripped. So sending a model's traffic to the gateway is only correct + * when the gateway's upstream *is* the endpoint that model would otherwise + * call. Redirecting an NVIDIA model into a gateway configured for + * `api.openai.com` does not degrade to "no spans"; it breaks the session. + * + * The launcher therefore passes the gateway's own upstreams + * (`NEMO_RELAY_PI_OPENAI_UPSTREAM`, `NEMO_RELAY_PI_ANTHROPIC_UPSTREAM`) and + * this module redirects only on a match. Unmatched models keep their own + * endpoint and produce no LLM spans, which is the honest outcome. + */ + +/** The API families the gateway serves, mapped to the upstream that backs each. */ +const SERVICEABLE_APIS: Record = { + 'openai-completions': 'openai', + 'openai-responses': 'openai', + 'anthropic-messages': 'anthropic', +}; + +export type RedirectConfig = { + /** Gateway base URL; the root, never root + `/v1`. */ + gatewayUrl: string; + /** What the gateway forwards OpenAI-compatible traffic to, when known. */ + openaiUpstream?: string; + /** What the gateway forwards Anthropic traffic to, when known. */ + anthropicUpstream?: string; + /** + * `match` (default) redirects only when the model's endpoint is the + * gateway's upstream. `force` skips that check for operators who know their + * gateway is correct but did not launch through `nemo-relay run`. `off` + * disables redirection entirely. + */ + mode: 'match' | 'force' | 'off'; +}; + +/** A model, narrowed to the fields redirection depends on. */ +export type RedirectModel = { + id: string; + api: string; + provider: string; + baseUrl: string; +}; + +/** + * Why a redirect did or did not happen. + * + * `code` exists so callers can tell a transient skip from one worth recording. + * `no-model` and `already-redirected` are bookkeeping -- a `model_select` is + * either coming or the work is done -- while the rest are the reasons a user + * ends up staring at a trace with no LLM spans, and belong in that trace. + */ +export type RedirectSkipCode = + | 'disabled' + | 'no-model' + | 'already-redirected' + | 'unserviceable-api' + | 'unknown-upstream' + | 'upstream-mismatch'; + +export type RedirectDecision = + | { kind: 'redirect'; provider: string; api: string; upstream: string; reason: string } + | { kind: 'skip'; code: RedirectSkipCode; provider?: string; api?: string; reason: string }; + +/** Whether this outcome explains something a trace reader would otherwise have to guess. */ +export function isNotable(decision: RedirectDecision): boolean { + return decision.kind === 'redirect' || !['no-model', 'already-redirected'].includes(decision.code); +} + +/** + * Normalize a base URL for comparison. + * + * Compares scheme, host, port and path with trailing slashes removed. A + * trailing `/v1` is *not* stripped: pi's Anthropic catalog entry is + * `https://api.anthropic.com` while its OpenAI entry is + * `https://api.openai.com/v1`, and the gateway's two defaults match those + * exactly, so treating `/v1` as noise would equate genuinely different + * endpoints on providers that host several API versions. + */ +export function normalizeBaseUrl(value: string): string { + const trimmed = value.trim().replace(/\/+$/, ''); + try { + const url = new URL(trimmed); + const path = url.pathname.replace(/\/+$/, ''); + return `${url.protocol}//${url.host.toLowerCase()}${path}`; + } catch { + return trimmed.toLowerCase(); + } +} + +/** + * Decide whether this model's provider should be pointed at the gateway. + * + * Pure, so the decision matrix is testable without a pi runtime. `redirected` + * carries providers already pointed at the gateway, which makes the call + * idempotent: once a provider is redirected, `model.baseUrl` reads back as the + * gateway and the upstream comparison would otherwise fail on the second call. + */ +export function decideRedirect( + model: RedirectModel | undefined, + config: RedirectConfig, + redirected: ReadonlySet, +): RedirectDecision { + if (config.mode === 'off') { + return { + kind: 'skip', + code: 'disabled', + reason: 'redirection disabled by NEMO_RELAY_PI_REDIRECT=off', + }; + } + if (!model) { + return { kind: 'skip', code: 'no-model', reason: 'no model selected yet' }; + } + if (redirected.has(model.provider)) { + return { + kind: 'skip', + code: 'already-redirected', + provider: model.provider, + api: model.api, + reason: 'already redirected', + }; + } + + const family = SERVICEABLE_APIS[model.api]; + if (!family) { + // Six of pi's providers speak an API the gateway has no route for + // (Bedrock, Azure OpenAI Responses, Google, Google Vertex, Mistral, OpenAI + // Codex). Redirecting them would 404 rather than degrade. + return { + kind: 'skip', + code: 'unserviceable-api', + provider: model.provider, + api: model.api, + reason: `the gateway serves no route for the ${model.api} API`, + }; + } + + const upstream = family === 'openai' ? config.openaiUpstream : config.anthropicUpstream; + + if (config.mode === 'force') { + return { + kind: 'redirect', + provider: model.provider, + api: model.api, + upstream: model.baseUrl, + reason: 'NEMO_RELAY_PI_REDIRECT=force; upstream match not checked', + }; + } + + if (!upstream) { + // Launched outside `nemo-relay run --agent pi`, so the gateway's upstream + // is unknown. Staying put is the safe default: a wrong redirect breaks the + // session, a skipped one only costs spans. + return { + kind: 'skip', + code: 'unknown-upstream', + provider: model.provider, + api: model.api, + reason: + `the gateway's ${family} upstream is unknown, so a redirect cannot be verified as safe; ` + + `launch through \`nemo-relay run --agent pi\`, or set NEMO_RELAY_PI_REDIRECT=force`, + }; + } + + if (normalizeBaseUrl(upstream) !== normalizeBaseUrl(model.baseUrl)) { + return { + kind: 'skip', + code: 'upstream-mismatch', + provider: model.provider, + api: model.api, + reason: + `model targets ${model.baseUrl} but the gateway forwards ${family} traffic to ${upstream}; ` + + `redirecting would send the request to the wrong provider`, + }; + } + + return { + kind: 'redirect', + provider: model.provider, + api: model.api, + upstream: model.baseUrl, + reason: `gateway forwards ${family} traffic to the same endpoint (${upstream})`, + }; +} + +/** Read redirection configuration from the environment the launcher sets. */ +export function redirectConfigFromEnv(gatewayUrl: string): RedirectConfig { + const raw = process.env.NEMO_RELAY_PI_REDIRECT; + const mode = raw === 'off' ? 'off' : raw === 'force' ? 'force' : 'match'; + // Spread conditionally rather than assigning `undefined`: the package builds + // under `exactOptionalPropertyTypes`, where an explicit `undefined` is not + // the same as an absent key. + const openaiUpstream = process.env.NEMO_RELAY_PI_OPENAI_UPSTREAM; + const anthropicUpstream = process.env.NEMO_RELAY_PI_ANTHROPIC_UPSTREAM; + return { + gatewayUrl, + mode, + ...(openaiUpstream ? { openaiUpstream } : {}), + ...(anthropicUpstream ? { anthropicUpstream } : {}), + }; +} diff --git a/integrations/pi/test/provider-redirect.test.mjs b/integrations/pi/test/provider-redirect.test.mjs new file mode 100644 index 000000000..c226dda8c --- /dev/null +++ b/integrations/pi/test/provider-redirect.test.mjs @@ -0,0 +1,169 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * The redirect decision matrix. + * + * `decideRedirect` is pure so this suite can cover every branch without a pi + * runtime. The branch that matters most is the mismatch guard: the gateway + * forwards to one statically configured upstream per API family, so a redirect + * is only correct when that upstream is the endpoint the model would otherwise + * have called. Getting this wrong does not cost spans, it breaks the session. + * + * Run: node --test integrations/pi/test/*.test.mjs + */ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +const { decideRedirect, isNotable, normalizeBaseUrl } = await import('../src/provider-redirect.ts'); + +const GATEWAY = 'http://127.0.0.1:4040'; + +/** The real NVIDIA catalog entry, which is what this was developed against. */ +const nvidiaModel = { + id: 'nvidia/nemotron-3-super-120b-a12b', + api: 'openai-completions', + provider: 'nvidia', + baseUrl: 'https://integrate.api.nvidia.com/v1', +}; + +const anthropicModel = { + id: 'claude-sonnet-4-5', + api: 'anthropic-messages', + provider: 'anthropic', + baseUrl: 'https://api.anthropic.com', +}; + +const matchConfig = (extra = {}) => ({ + gatewayUrl: GATEWAY, + mode: 'match', + openaiUpstream: 'https://integrate.api.nvidia.com/v1', + anthropicUpstream: 'https://api.anthropic.com', + ...extra, +}); + +const none = new Set(); + +describe('redirect decision', () => { + it('redirects when the gateway forwards to the model’s own endpoint', () => { + const decision = decideRedirect(nvidiaModel, matchConfig(), none); + assert.equal(decision.kind, 'redirect'); + assert.equal(decision.provider, 'nvidia'); + assert.equal(decision.upstream, 'https://integrate.api.nvidia.com/v1'); + }); + + // The failure this guard exists for: pi's catalog says NVIDIA, the gateway + // forwards to OpenAI, and redirecting would send the request to a provider + // that has never heard of the model or the key. + it('refuses when the gateway forwards somewhere else', () => { + const decision = decideRedirect( + nvidiaModel, + matchConfig({ openaiUpstream: 'https://api.openai.com/v1' }), + none, + ); + assert.equal(decision.kind, 'skip'); + assert.equal(decision.code, 'upstream-mismatch'); + assert.match(decision.reason, /wrong provider/); + }); + + it('picks the upstream matching the model’s API family, not the other one', () => { + // Anthropic model, anthropic upstream matches, openai upstream does not. + const decision = decideRedirect( + anthropicModel, + matchConfig({ openaiUpstream: 'https://api.openai.com/v1' }), + none, + ); + assert.equal(decision.kind, 'redirect'); + assert.equal(decision.api, 'anthropic-messages'); + }); + + it('refuses an API the gateway has no route for', () => { + for (const api of ['google-generative-ai', 'bedrock-converse-stream', 'mistral-conversations']) { + const decision = decideRedirect({ ...nvidiaModel, api }, matchConfig(), none); + assert.equal(decision.kind, 'skip', api); + assert.equal(decision.code, 'unserviceable-api', api); + } + }); + + // Launched outside `nemo-relay run --agent pi`, so nothing told the extension + // what the gateway fronts. Staying put costs spans; guessing costs the session. + it('refuses when the gateway upstream is unknown', () => { + const decision = decideRedirect( + nvidiaModel, + { gatewayUrl: GATEWAY, mode: 'match' }, + none, + ); + assert.equal(decision.kind, 'skip'); + assert.equal(decision.code, 'unknown-upstream'); + assert.match(decision.reason, /NEMO_RELAY_PI_REDIRECT=force/); + }); + + it('force skips the match check, match does not', () => { + const forced = decideRedirect( + nvidiaModel, + { gatewayUrl: GATEWAY, mode: 'force', openaiUpstream: 'https://api.openai.com/v1' }, + none, + ); + assert.equal(forced.kind, 'redirect'); + assert.match(forced.reason, /not checked/); + }); + + it('off disables redirection entirely', () => { + const decision = decideRedirect(nvidiaModel, matchConfig({ mode: 'off' }), none); + assert.equal(decision.kind, 'skip'); + assert.equal(decision.code, 'disabled'); + }); + + it('has nothing to decide before a model is resolved', () => { + const decision = decideRedirect(undefined, matchConfig(), none); + assert.equal(decision.kind, 'skip'); + assert.equal(decision.code, 'no-model'); + }); + + // Without this, the second call compares the gateway URL against the upstream + // and reports a mismatch for a provider it redirected itself. + it('is idempotent once a provider has been redirected', () => { + const decision = decideRedirect(nvidiaModel, matchConfig(), new Set(['nvidia'])); + assert.equal(decision.kind, 'skip'); + assert.equal(decision.code, 'already-redirected'); + }); +}); + +describe('what reaches the trace', () => { + // A mark per session for "no model yet" is noise; a mark explaining why LLM + // spans are absent is the whole point. + it('records outcomes that explain a trace, not bookkeeping', () => { + assert.equal(isNotable(decideRedirect(nvidiaModel, matchConfig(), none)), true); + assert.equal( + isNotable(decideRedirect(nvidiaModel, matchConfig({ openaiUpstream: 'https://x.test' }), none)), + true, + ); + assert.equal(isNotable(decideRedirect(undefined, matchConfig(), none)), false); + assert.equal(isNotable(decideRedirect(nvidiaModel, matchConfig(), new Set(['nvidia']))), false); + }); +}); + +describe('base URL comparison', () => { + it('ignores trailing slashes and host case', () => { + assert.equal( + normalizeBaseUrl('https://API.Nvidia.com/v1/'), + normalizeBaseUrl('https://api.nvidia.com/v1'), + ); + }); + + // `/v1` is a real path segment, not noise: providers host several API + // versions, and equating them would redirect into the wrong one. + it('does not treat a /v1 suffix as equivalent to its absence', () => { + assert.notEqual( + normalizeBaseUrl('https://api.anthropic.com'), + normalizeBaseUrl('https://api.anthropic.com/v1'), + ); + }); + + it('keeps ports distinct', () => { + assert.notEqual( + normalizeBaseUrl('http://127.0.0.1:4040'), + normalizeBaseUrl('http://127.0.0.1:4141'), + ); + }); +}); From 184b416e103b5157552ad2c1c838cb5fd0cc4501 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 16:45:44 -0700 Subject: [PATCH 10/41] docs(pi): drop tracker links from the extension README The Status section linked out to an issue tracker that is not readable from this repository, and named its identifiers inline. Neither belongs in a public README: a reader outside the org gets dead links, and the identifiers carry no meaning for them. Says the same thing in prose instead. Nothing about the described behaviour changes -- it is still a proof of concept verified against pi v0.84.0, and model redirection is still conditional on the gateway fronting the model's provider. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- integrations/pi/README.md | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/integrations/pi/README.md b/integrations/pi/README.md index da37bb7f0..3721841c9 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -16,19 +16,14 @@ all policy and all span construction happen in the gateway. ## Status -Proof of concept, tracked under -[RELAY-727](https://linear.app/nvidia/issue/RELAY-727), -[RELAY-728](https://linear.app/nvidia/issue/RELAY-728), -[RELAY-729](https://linear.app/nvidia/issue/RELAY-729) and -[RELAY-730](https://linear.app/nvidia/issue/RELAY-730). Verified against pi -`v0.84.0`. pi ships breaking changes through *minor* releases and has no -major-release channel, so re-verify hook signatures before relying on them. - -**Model traffic is redirected conditionally** -([RELAY-732](https://linear.app/nvidia/issue/RELAY-732)). pi has no base-URL -flag and no generic environment override — it resolves `baseUrl` per model from -a generated catalog — so the extension points the active model's provider at the -gateway itself. It only does so when the gateway forwards to the endpoint that +Proof of concept. Verified against pi `v0.84.0`. pi ships breaking changes +through *minor* releases and has no major-release channel, so re-verify hook +signatures before relying on them. + +**Model traffic is redirected conditionally.** pi has no base-URL flag and no +generic environment override — it resolves `baseUrl` per model from a generated +catalog — so the extension points the active model's provider at the gateway +itself. It only does so when the gateway forwards to the endpoint that model would otherwise have called; see [Model redirection](#model-redirection). When it does not, you get tool and turn activity but no LLM spans. From 4410b3a62b36f8bb20ba3548e33298a00049e1fd Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 17:09:24 -0700 Subject: [PATCH 11/41] feat(pi): apply gateway argument transforms to pi tool calls The transform half of the pi tool-policy work. Guardrails could already block a call; a request intercept could not change one, because the hook verdict travelled only as an HTTP status code and there was no channel for a rewritten payload. The chain already existed. `tool_request_intercepts(name, args) -> Result` is public in core and returns rewritten arguments; `start_tool` ran the guardrail chain and never this one. So the gateway side is wiring: run the chain, use its output as the span's arguments so the trace records what will execute, and hand it back. Handing it back needed one plumbing change. `pi_hook` builds its response before `apply_events` runs, so `apply_events` now returns `HookEffects` carrying the rewrite, and the pi adapter merges it into the body: {"tool_call": {"tool_call_id": "...", "input": {...}}} Absent a rewrite the body stays `{}`, which is what an allow has always been, so an older extension is unaffected. `tool_call_id` is echoed so the extension can refuse a body belonging to a different call. Gated per agent. Codex and Claude Code have no way to execute a rewrite, so running the chain for them would record arguments on the span that never ran -- worse than not running it, because the trace would then disagree with reality. The extension constrains the rewrite rather than validating it, because it cannot validate it. pi validates arguments before the `tool_call` hook and never re-validates -- its own types say so -- and the extension cannot read a built-in tool's schema: pi exposes `tools` only on the `Extension` interface, which is an extension's own registered tools. Of the three options the design considered (fetch the schema, forward the schema, constrain the transform), the first two are therefore impossible and the third is forced. So a transform may rewrite the values of existing keys, preserving each value's JSON type, recursively. Adding a key, removing one, changing a type or changing an array's length is refused, which keeps the required keys and types the schema already accepted. This is structural, not schema validation: pattern, enum and range constraints are not checked and cannot be, and that limitation is documented and asserted rather than glossed. A refused transform blocks the call. Running the original arguments would silently discard a policy decision, which is the failure the transform existed to prevent, and it is a different axis from NEMO_RELAY_PI_FAIL, which governs an unreachable gateway rather than one that answered with something unusable. Verified against live pi v0.84.0 twice. A shape-preserving rewrite of a read path executed: the model asked for alpha.txt, the gateway rewrote it to beta.txt, and pi read beta.txt. A key-adding rewrite blocked every one of eight tool calls, and the model reported it as a policy misconfiguration rather than a refusal of its request, which is what the reason string is written to produce. An earlier run of that second check appeared to pass the unsafe transform through. It had not: the stub only rewrote paths containing alpha.txt, so when the block worked the model retried with `cat alpha.txt` through bash, which the stub left alone. The test was wrong, not the code -- logging every call rather than only the rewritten ones showed it immediately. Green: 1166 + 12 + 102 Rust, 52 Node, tsc clean, docs-linkcheck 0 errors, clippy -D warnings clean, pre-commit clean apart from cargo-deny/gofmt/go-vet, which are not installed on this machine. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- crates/cli/src/agents/pi/adapter.rs | 31 ++++ crates/cli/src/events/mod.rs | 11 ++ crates/cli/src/server/mod.rs | 8 +- crates/cli/src/sessions/mod.rs | 84 ++++++++-- crates/cli/src/sessions/routing.rs | 18 +- .../cli/tests/coverage/shared/server_tests.rs | 108 +++++++++++- docs/nemo-relay-cli/pi.mdx | 27 +++ integrations/pi/README.md | 38 +++++ integrations/pi/index.ts | 30 +++- integrations/pi/src/argument-transform.ts | 158 ++++++++++++++++++ integrations/pi/src/gateway-client.ts | 25 ++- .../pi/test/argument-transform.test.mjs | 125 ++++++++++++++ integrations/pi/test/gateway-client.test.mjs | 5 +- 13 files changed, 639 insertions(+), 29 deletions(-) create mode 100644 integrations/pi/src/argument-transform.ts create mode 100644 integrations/pi/test/argument-transform.test.mjs diff --git a/crates/cli/src/agents/pi/adapter.rs b/crates/cli/src/agents/pi/adapter.rs index e91dbc426..0e50e5b37 100644 --- a/crates/cli/src/agents/pi/adapter.rs +++ b/crates/cli/src/agents/pi/adapter.rs @@ -8,6 +8,7 @@ use crate::agents::shared::adapters::{ AdapterOutcome, ClassificationRules, PI_PAYLOAD_EXTRACTOR, classify, }; use crate::events::AgentKind; +use crate::sessions::HookEffects; /// Normalizes pi extension hook payloads and returns the response the extension expects. /// @@ -68,3 +69,33 @@ pub(crate) fn adapt(payload: Value, headers: &HeaderMap) -> AdapterOutcome { response: json!({}), } } + +/// Merge anything the session layer produced into the hook response body. +/// +/// The wire contract the extension implements: +/// +/// - `{}` -- allow, arguments unchanged. Every hook that is not a gated `tool_call` returns this, +/// and so does a `tool_call` no intercept rewrote. +/// - `{"tool_call": {"tool_call_id": "...", "input": {...}}}` -- allow, but execute *these* +/// arguments instead. The extension applies them in place onto pi's `event.input`. +/// - HTTP 403 with `error.type = "nemo_relay_guardrail_rejected"` -- block, reason verbatim. +/// Produced by `CliError::into_response`, not here. +/// +/// `tool_call_id` is echoed so the extension can refuse a body that does not belong to the call it +/// just posted, rather than applying someone else's arguments. +pub(crate) fn response_with_effects(response: Value, effects: &HookEffects) -> Value { + let Some(transform) = effects.tool_argument_transform.as_ref() else { + return response; + }; + let mut response = response; + if let Some(object) = response.as_object_mut() { + object.insert( + "tool_call".into(), + json!({ + "tool_call_id": transform.tool_call_id, + "input": transform.arguments, + }), + ); + } + response +} diff --git a/crates/cli/src/events/mod.rs b/crates/cli/src/events/mod.rs index 0535249d6..802617711 100644 --- a/crates/cli/src/events/mod.rs +++ b/crates/cli/src/events/mod.rs @@ -40,6 +40,17 @@ impl AgentKind { pub(crate) const fn has_explicit_turn_start(self) -> bool { matches!(self, Self::Pi) } + + // Whether this harness can execute arguments the gateway rewrote. + // + // Only pi: its `tool_call` hook documents in-place mutation of `input` as the mechanism, and + // the extension applies whatever the hook response carries. Codex and Claude Code have no + // equivalent return path, so running the request-intercept chain for them would record + // arguments on the tool span that never executed -- worse than not running it, because the + // trace would then disagree with reality. + pub(crate) const fn applies_tool_argument_transforms(self) -> bool { + matches!(self, Self::Pi) + } } #[derive(Debug, Clone, PartialEq)] diff --git a/crates/cli/src/server/mod.rs b/crates/cli/src/server/mod.rs index 678e59f9f..066044917 100644 --- a/crates/cli/src/server/mod.rs +++ b/crates/cli/src/server/mod.rs @@ -1220,11 +1220,15 @@ async fn pi_hook( state.touch(); let Json(payload) = payload.map_err(hook_payload_rejection)?; let outcome = pi::adapt(payload, &headers); - state + let effects = state .sessions .apply_events(&headers, outcome.events) .await?; - Ok(Json(outcome.response)) + // pi is the one agent whose hook response can carry a rewritten payload back: its `tool_call` + // hook documents in-place mutation of `input`, so the extension can apply what a request + // intercept produced. Absent a rewrite the body stays `{}`, which is what an allow has always + // been, so an older extension keeps working unchanged. + Ok(Json(pi::response_with_effects(outcome.response, &effects))) } fn hook_payload_rejection(rejection: JsonRejection) -> CliError { diff --git a/crates/cli/src/sessions/mod.rs b/crates/cli/src/sessions/mod.rs index b031c5df2..1536cc379 100644 --- a/crates/cli/src/sessions/mod.rs +++ b/crates/cli/src/sessions/mod.rs @@ -18,7 +18,7 @@ use nemo_relay::api::scope::{ }; use nemo_relay::api::tool::{ ToolCallEndParams, ToolCallParams, ToolHandle, tool_call, tool_call_end, - tool_conditional_execution, + tool_conditional_execution, tool_request_intercepts, }; use serde_json::{Map, Value, json}; use tokio::sync::Mutex; @@ -60,6 +60,27 @@ const ROUTING_IDENTITY_HEADERS: &[&str] = &[ "x-nemo-relay-source", ]; +/// Arguments a request intercept rewrote, on their way back to the agent that will execute them. +/// +/// The gateway cannot apply a transform itself -- it never runs the tool -- so the rewrite has to +/// travel back in the hook response and be applied by the extension. One pi hook post carries at +/// most one `tool_call`, so a single value is enough; `tool_call_id` lets the receiver assert the +/// response belongs to the call it just sent. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ToolArgumentTransform { + pub(crate) tool_call_id: String, + pub(crate) arguments: Value, +} + +/// What one batch of hook events produced that the HTTP response still has to carry. +/// +/// Empty for every agent except pi, and empty for pi unless a request intercept actually changed +/// the arguments. +#[derive(Debug, Clone, Default, PartialEq)] +pub(crate) struct HookEffects { + pub(crate) tool_argument_transform: Option, +} + #[derive(Clone)] pub(crate) struct SessionManager { inner: Arc>>, @@ -153,6 +174,9 @@ fn insert_routing_identity_header(headers: &mut Map, name: &str, pub(super) struct Session { agent_kind: AgentKind, session_id: String, + /// Set by `start_tool` when a request intercept rewrote the arguments; taken by the routing + /// layer once the event batch has been applied, so the hook response can carry it back. + tool_argument_transform: Option, scope_stack: ScopeStackHandle, session_started: bool, session_metadata: Value, @@ -339,7 +363,8 @@ impl SessionManager { &self, headers: &HeaderMap, events: Vec, - ) -> Result<(), CliError> { + ) -> Result { + let mut effects = HookEffects::default(); let mut subscriber_deliveries = Vec::new(); let mut alignment_state = self.alignment.lock().await; let mut sessions = self.inner.lock().await; @@ -363,18 +388,22 @@ impl SessionManager { continue; }; let event_kind = event_agent_kind(&event); - let (should_remove_session, subscriber_delivery) = apply_event_to_session( - &mut sessions, - &session_id, - event, - event_kind, - config.clone(), - is_agent_started, - ) - .await?; + let (should_remove_session, subscriber_delivery, tool_argument_transform) = + apply_event_to_session( + &mut sessions, + &session_id, + event, + event_kind, + config.clone(), + is_agent_started, + ) + .await?; if let Some(subscriber_delivery) = subscriber_delivery { subscriber_deliveries.push(subscriber_delivery); } + if tool_argument_transform.is_some() { + effects.tool_argument_transform = tool_argument_transform; + } if is_agent_started { // A just-opened parent may unlock one or more child SessionStart hooks that arrived // earlier in this batch or an earlier request. @@ -395,7 +424,7 @@ impl SessionManager { for subscriber_delivery in subscriber_deliveries { subscriber_delivery.wait().await?; } - Ok(()) + Ok(effects) } /// Legacy manual-lifecycle entry point retained for tests that drive correlation behavior @@ -702,6 +731,7 @@ impl Session { Self { agent_kind, session_id, + tool_argument_transform: None, scope_stack: create_scope_stack(), session_started: false, session_metadata: Value::Null, @@ -1514,10 +1544,30 @@ impl Session { } else { event.arguments }; + tool_conditional_execution(event.tool_name.as_str(), &arguments).await?; + + // Guardrails decide whether the call runs; request intercepts decide what it runs with. + // Only worth running for a harness that can actually execute the rewrite -- see + // `AgentKind::applies_tool_argument_transforms`. The transformed arguments become the + // span's arguments too, so the trace records what will execute rather than what was + // proposed. + let arguments = if self.agent_kind.applies_tool_argument_transforms() { + let transformed = + tool_request_intercepts(event.tool_name.as_str(), arguments.clone()).await?; + if transformed != arguments { + self.tool_argument_transform = Some(ToolArgumentTransform { + tool_call_id: event.tool_call_id.clone(), + arguments: transformed.clone(), + }); + } + transformed + } else { + arguments + }; + let active_tool_arguments = arguments.clone(); let active_tool_name = event.tool_name.clone(); let active_tool_owner_subagent_id = owner.subagent_id.clone(); - tool_conditional_execution(event.tool_name.as_str(), &arguments).await?; let metadata = tool_correlation_metadata( self.event_identity_metadata(event.metadata), owner.status, @@ -1652,6 +1702,14 @@ impl Session { .filter(|subagent_id| self.subagents.contains_key(subagent_id)) } + /// Hand over any rewrite produced while applying this batch, clearing it. + /// + /// Taken rather than read so a later hook on the same session cannot re-apply a stale + /// transform to a different tool call. + fn take_tool_argument_transform(&mut self) -> Option { + self.tool_argument_transform.take() + } + // Emits a mark event after ensuring an enclosing scope exists. Generic and unknown hooks use // this path so unsupported agent events remain visible without changing scope structure. // diff --git a/crates/cli/src/sessions/routing.rs b/crates/cli/src/sessions/routing.rs index 4a2aa6d81..96f285c31 100644 --- a/crates/cli/src/sessions/routing.rs +++ b/crates/cli/src/sessions/routing.rs @@ -15,7 +15,7 @@ use crate::configuration::SessionConfig; use crate::error::CliError; use crate::events::{AgentKind, NormalizedEvent, SessionEvent}; -use super::{LlmGatewayStart, Session}; +use super::{LlmGatewayStart, Session, ToolArgumentTransform}; pub(super) fn apply_start_alias(start: &mut LlmGatewayStart, alias: &SessionAlias) { start.session_id = Some(alias.parent_session_id.clone()); @@ -56,7 +56,14 @@ pub(super) async fn apply_event_to_session( event_kind: AgentKind, config: SessionConfig, is_agent_started: bool, -) -> Result<(bool, Option), CliError> { +) -> Result< + ( + bool, + Option, + Option, + ), + CliError, +> { let session = sessions .entry(session_id.to_string()) .or_insert_with(|| Session::new(session_id.to_string(), event_kind, config)); @@ -67,7 +74,12 @@ pub(super) async fn apply_event_to_session( session.agent_kind = event_kind; } let subscriber_delivery = session.apply(event).await?; - Ok((session.is_empty(), subscriber_delivery)) + let tool_argument_transform = session.take_tool_argument_transform(); + Ok(( + session.is_empty(), + subscriber_delivery, + tool_argument_transform, + )) } pub(super) async fn promote_pending_subagents_for_parent( diff --git a/crates/cli/tests/coverage/shared/server_tests.rs b/crates/cli/tests/coverage/shared/server_tests.rs index 78968b77b..923e6b2f8 100644 --- a/crates/cli/tests/coverage/shared/server_tests.rs +++ b/crates/cli/tests/coverage/shared/server_tests.rs @@ -19,9 +19,10 @@ use nemo_relay::api::llm::LlmRequestInterceptOutcome; use nemo_relay::api::registry::{ deregister_llm_execution_intercept, deregister_llm_request_intercept, deregister_llm_stream_execution_intercept, deregister_scope_sanitize_end_guardrail, - deregister_tool_conditional_execution_guardrail, register_llm_execution_intercept, - register_llm_request_intercept, register_llm_stream_execution_intercept, - register_scope_sanitize_end_guardrail, register_tool_conditional_execution_guardrail, + deregister_tool_conditional_execution_guardrail, deregister_tool_request_intercept, + register_llm_execution_intercept, register_llm_request_intercept, + register_llm_stream_execution_intercept, register_scope_sanitize_end_guardrail, + register_tool_conditional_execution_guardrail, register_tool_request_intercept, }; use nemo_relay::api::subscriber::{deregister_subscriber, flush_subscribers, register_subscriber}; use nemo_relay::plugin::dynamic::DynamicPluginKind; @@ -4507,3 +4508,104 @@ async fn gateway_streaming_hit_carries_event_stream_content_type() { handle.await.unwrap().unwrap(); let _ = nemo_relay::plugin::clear_plugin_configuration(); } + +struct ToolInterceptCleanup(&'static str); + +impl Drop for ToolInterceptCleanup { + fn drop(&mut self) { + let _ = deregister_tool_request_intercept(self.0); + } +} + +// The gateway cannot apply a rewrite -- it never runs the tool -- so a request intercept is only +// useful to pi if its output travels back in the hook response. This asserts the whole path: the +// chain runs on the hook, the rewrite reaches the body, and the tool_call_id is echoed so the +// extension can tell the response belongs to the call it posted. +#[tokio::test] +async fn pi_tool_call_hook_returns_arguments_a_request_intercept_rewrote() { + let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; + let _ = deregister_tool_request_intercept("cli-pi-redactor"); + register_tool_request_intercept( + "cli-pi-redactor", + 1, + // Do not break the chain: a later intercept must still get to see the rewrite. + false, + Arc::new(|_name: String, args: Value| { + Box::pin(async move { + let mut args = args; + if let Some(object) = args.as_object_mut() + && object.get("path").and_then(Value::as_str) == Some("/work/.env") + { + object.insert("path".into(), json!("/work/.env.example")); + } + Ok(args) + }) + }), + ) + .unwrap(); + let _cleanup = ToolInterceptCleanup("cli-pi-redactor"); + + let app = router(test_config()); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/hooks/pi") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "session_id": "pi-transform-session", + "hook_event_name": "tool_call", + "tool_call_id": "call-transform", + "tool_name": "read", + "input": { "path": "/work/.env" } + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["tool_call"]["tool_call_id"], json!("call-transform")); + assert_eq!( + body["tool_call"]["input"], + json!({ "path": "/work/.env.example" }) + ); +} + +// An unchanged call must keep returning the bare `{}` an allow has always been, or every existing +// extension would start seeing a payload it has no contract for. +#[tokio::test] +async fn pi_tool_call_hook_omits_the_transform_when_nothing_rewrote_the_arguments() { + let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; + let app = router(test_config()); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/hooks/pi") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "session_id": "pi-no-transform-session", + "hook_event_name": "tool_call", + "tool_call_id": "call-plain", + "tool_name": "read", + "input": { "path": "/work/README.md" } + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body, json!({})); +} diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index 8402f780c..3f677c294 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -135,6 +135,33 @@ The default is fail-open, so an unreachable gateway does not brick the agent. explicitly that it is an infrastructure fault rather than a judgment about the request. +## Argument Transforms + +A Relay request intercept can rewrite a tool call's arguments. The gateway never +executes the tool, so the rewrite travels back in the allow response and the +extension applies it to pi's `event.input` in place. + +| Response body | Meaning | +| --- | --- | +| `{}` | Allow, arguments unchanged | +| `{"tool_call": {"tool_call_id": "…", "input": {…}}}` | Allow, but execute these arguments | + + +The rewrite is **constrained, not validated**. pi validates tool arguments +before the hook and never re-validates, and the extension cannot read a built-in +tool's schema, so the extension enforces a shape invariant instead: a transform +may rewrite the values of existing keys, preserving each value's JSON type. It +may not add or remove keys, change a type, or change an array's length. + +That keeps the required keys and types intact, but it is not schema validation — +`pattern`, `enum` and range constraints are not checked. A transform that +rewrites a value to one the schema would reject will still execute. + + +A transform that violates the invariant **blocks the call**, with a reason +stating that a policy could not be applied rather than that the request was +refused. Running the original arguments would silently discard the policy. + ## Captured Events The extension posts 11 hooks. pi reports both ends of a turn, so Relay opens the diff --git a/integrations/pi/README.md b/integrations/pi/README.md index 3721841c9..fb90c8b2d 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -74,6 +74,44 @@ The block reason reaches the model **verbatim**: pi hands it to as error codes — a reason that says what to do instead produces a model that adapts rather than one that gives up. +## Argument transforms + +A Relay **request intercept** can rewrite a tool's arguments. The gateway never +runs the tool, so the rewrite comes back in the allow response and the extension +applies it to pi's `event.input` **in place** — which is what pi documents as +the mechanism, and is required: pi hands the same object to the tool and to +later handlers, so replacing the reference would be discarded. + +| Response body | Meaning | +|---|---| +| `{}` | Allow, arguments unchanged. Every non-gated hook, and any `tool_call` no intercept rewrote | +| `{"tool_call": {"tool_call_id": "…", "input": {…}}}` | Allow, but execute *these* arguments | + +⚠️ **The rewrite is constrained, not validated.** pi validates arguments +*before* the `tool_call` hook and never re-validates — its own types say "no +re-validation is performed after mutation" — so a rewrite that violates the +tool's schema would execute. The extension cannot check it against the schema +either: pi exposes `tools` only on the `Extension` interface, which is an +extension's *own* registered tools, so there is no way to read the schema of a +built-in like `read`. + +So the extension enforces a **shape** invariant instead: a transform may rewrite +the values of existing keys, preserving each value's JSON type, recursively. +Adding a key, removing a key, changing a type, or changing an array's length is +refused. An argument object that satisfied the schema before therefore still has +the required keys of the required types afterwards. + +**This is structural, not schema validation.** `pattern`, `enum`, `minimum` and +`format` are not checked and cannot be. A transform that rewrites a string to +one the schema would reject still executes. + +A refused transform **blocks the call**, with a reason that says the policy could +not be applied rather than that the request was refused. Running the original +arguments instead would silently discard a policy decision, which is the failure +the transform existed to prevent. This is a different axis from +`NEMO_RELAY_PI_FAIL`, which governs an unreachable gateway rather than one that +answered with something unusable. + ## Model redirection pi resolves a base URL per model from a generated catalog, so there is no flag or diff --git a/integrations/pi/index.ts b/integrations/pi/index.ts index 44ea5b2a3..b0ea24233 100644 --- a/integrations/pi/index.ts +++ b/integrations/pi/index.ts @@ -57,6 +57,11 @@ import { postHook, resolveFault, } from './src/gateway-client.ts'; +import { + applyTransform, + decideTransform, + refusalReason, +} from './src/argument-transform.ts'; import { type RedirectConfig, decideRedirect, @@ -416,10 +421,31 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { const decision = outcome.kind === 'fault' ? resolveFault(active, outcome.detail, event.toolName) : outcome; + if (decision.kind === 'block') return { block: true, reason: decision.reason }; + + // Allowed. A request intercept may still have rewritten the arguments, in which case they + // arrive in the response body and pi expects them applied to `event.input` in place. + if (decision.kind === 'allow' && decision.body) { + const transform = decideTransform(decision.body, event.toolCallId, event.input); + if (transform.kind === 'refuse') { + // Blocking, not falling back: running the original arguments would discard a policy + // decision. Distinct from a guardrail rejection, and the reason says so. + return { block: true, reason: refusalReason(event.toolName, transform.reason) }; + } + if (transform.kind === 'apply') { + applyTransform(event.input, transform.input); + emit(ctx, { + hook_event_name: 'tool_arguments_transformed', + tool_call_id: event.toolCallId, + tool_name: event.toolName, + ...attribution(), + }); + } + } + // `undefined` is the only correct allow value: a truthy result without // `block` is inert but overwrites earlier handlers' results. - if (decision.kind !== 'block') return undefined; - return { block: true, reason: decision.reason }; + return undefined; }, ); diff --git a/integrations/pi/src/argument-transform.ts b/integrations/pi/src/argument-transform.ts new file mode 100644 index 000000000..23ef2bcf9 --- /dev/null +++ b/integrations/pi/src/argument-transform.ts @@ -0,0 +1,158 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Applies gateway-rewritten tool arguments to pi's `tool_call` event. + * + * A Relay request intercept can rewrite a tool's arguments. The gateway never + * runs the tool, so the rewrite comes back in the hook response and this module + * applies it to `event.input`. + * + * **Two pi facts make that dangerous, and shape everything here.** + * + * 1. *pi validates before the hook and never re-validates.* + * `validateToolArguments` returns a `structuredClone`, that clone is what + * `tool_call` receives, and pi's own types say "later `tool_call` handlers + * see earlier mutations. No re-validation is performed after mutation." So + * arguments that violate the tool's schema will execute. + * 2. *The extension cannot see the schema.* pi exposes `tools` only on the + * `Extension` interface, which is an extension's own registered tools -- + * there is no accessor for the schema of a built-in like `read` or `bash`. + * Neither we nor the gateway can check the rewrite against it. + * + * So the transform is constrained instead of validated: it may **rewrite the + * values of existing keys, preserving each value's JSON type**, recursively. + * Adding a key, removing a key, changing a type, or changing an array's length + * is refused. An object that satisfied the schema before therefore still has + * the required keys, of the required types, afterwards. + * + * ⚠️ **This is a structural guarantee, not schema validation.** Value-level + * constraints -- `pattern`, `enum`, `minimum`, `format` -- are not checked and + * cannot be. A transform that rewrites a string to one the schema would reject + * still executes. + * + * A refused transform **blocks the call**. Running the original arguments would + * silently discard a policy decision, which is the failure the transform + * existed to prevent; and this is a different axis from + * `NEMO_RELAY_PI_FAIL`, which governs an unreachable gateway rather than a + * gateway that answered with something unusable. + */ + +/** What the gateway sent back on an allowed `tool_call`. */ +export type TransformEnvelope = { + tool_call_id?: unknown; + input?: unknown; +}; + +export type TransformOutcome = + | { kind: 'none' } + | { kind: 'apply'; input: Record } + | { kind: 'refuse'; reason: string }; + +/** JSON type name used for the type-preservation check. `null` is its own type. */ +function jsonType(value: unknown): string { + if (value === null) return 'null'; + if (Array.isArray(value)) return 'array'; + return typeof value; +} + +/** + * Check that `next` preserves the shape of `current`. + * + * Returns null when the shape holds, or a human-readable reason when it does + * not. The reason reaches the model verbatim through pi, so it names the path. + */ +export function shapeViolation(current: unknown, next: unknown, path = 'input'): string | null { + const currentType = jsonType(current); + const nextType = jsonType(next); + if (currentType !== nextType) { + return `${path} changed type from ${currentType} to ${nextType}`; + } + if (currentType === 'object') { + const currentKeys = Object.keys(current as object).sort(); + const nextKeys = Object.keys(next as object).sort(); + const added = nextKeys.filter((key) => !currentKeys.includes(key)); + const removed = currentKeys.filter((key) => !nextKeys.includes(key)); + if (added.length > 0) return `${path} added ${added.join(', ')}`; + if (removed.length > 0) return `${path} removed ${removed.join(', ')}`; + for (const key of currentKeys) { + const violation = shapeViolation( + (current as Record)[key], + (next as Record)[key], + `${path}.${key}`, + ); + if (violation) return violation; + } + return null; + } + if (currentType === 'array') { + const currentItems = current as unknown[]; + const nextItems = next as unknown[]; + if (currentItems.length !== nextItems.length) { + return `${path} changed length from ${currentItems.length} to ${nextItems.length}`; + } + for (const [index, item] of currentItems.entries()) { + const violation = shapeViolation(item, nextItems[index], `${path}[${index}]`); + if (violation) return violation; + } + } + return null; +} + +/** + * Decide what to do with a hook response body for a given tool call. + * + * Pure, so the decision matrix is testable without pi or a gateway. + */ +export function decideTransform( + body: { tool_call?: TransformEnvelope } | null, + toolCallId: string, + current: Record, +): TransformOutcome { + const envelope = body?.tool_call; + if (!envelope || envelope.input === undefined) return { kind: 'none' }; + + // A body for a different call means the gateway and the extension disagree about what is in + // flight. Applying it would rewrite one tool call with another's arguments. + if (typeof envelope.tool_call_id === 'string' && envelope.tool_call_id !== toolCallId) { + return { + kind: 'refuse', + reason: `the transform names tool call ${envelope.tool_call_id}, not ${toolCallId}`, + }; + } + + if (jsonType(envelope.input) !== 'object') { + return { kind: 'refuse', reason: `the transform is ${jsonType(envelope.input)}, not an object` }; + } + + const violation = shapeViolation(current, envelope.input); + if (violation) return { kind: 'refuse', reason: violation }; + + return { kind: 'apply', input: envelope.input as Record }; +} + +/** + * Apply a transform to pi's event object **in place**. + * + * In place is required, not stylistic: pi passes the same object on to the tool + * and to later handlers, so replacing the reference would be discarded. Because + * the shape check has already run, this only ever overwrites existing keys. + */ +export function applyTransform( + target: Record, + input: Record, +): void { + for (const [key, value] of Object.entries(input)) { + target[key] = value; + } +} + +/** The block reason used when a transform arrives but cannot be applied safely. */ +export function refusalReason(toolName: string, detail: string): string { + return ( + `A NeMo Relay policy rewrote the arguments for this ${toolName} call, but the rewrite could ` + + `not be applied safely: ${detail}. The call was blocked rather than run with the original ` + + `arguments, because running them would ignore the policy. This is a configuration problem in ` + + `the policy, not a judgement about your request.` + ); +} diff --git a/integrations/pi/src/gateway-client.ts b/integrations/pi/src/gateway-client.ts index 7e9036ae0..b5b8ca4f1 100644 --- a/integrations/pi/src/gateway-client.ts +++ b/integrations/pi/src/gateway-client.ts @@ -6,7 +6,10 @@ * * The wire contract, verified against `crates/cli`: * - * - **Allow** is any 2xx. The adapter returns `{}`; the body is not meaningful. + * - **Allow** is any 2xx. The body is `{}` unless a request intercept rewrote + * the arguments, in which case it carries + * `{"tool_call": {"tool_call_id": "...", "input": {...}}}` and the caller is + * expected to execute those arguments instead. See `argument-transform.ts`. * - **Block** is `403` with * `{"error": {"type": "nemo_relay_guardrail_rejected", "reason": ""}}`. * The rejection comes from the tool conditional-execution guardrail chain that @@ -23,7 +26,8 @@ /** Outcome of posting one hook to the gateway. */ export type HookOutcome = - | { kind: 'allow' } + /** Allowed. `body` carries a rewritten payload when a request intercept produced one. */ + | { kind: 'allow'; body?: { tool_call?: { tool_call_id?: unknown; input?: unknown } } } | { kind: 'block'; reason: string } | { kind: 'fault'; detail: string }; @@ -79,7 +83,12 @@ export async function postHook( signal: controller.signal, }); - if (response.ok) return { kind: 'allow' }; + if (response.ok) { + // An allow body is `{}` unless a request intercept rewrote the arguments, so parsing is + // best effort: a body we cannot read is still an allow, just one with nothing to apply. + const body = await safeJson(response); + return body && typeof body === 'object' ? { kind: 'allow', body } : { kind: 'allow' }; + } if (response.status === 403) { const body = await safeJson(response); @@ -133,9 +142,15 @@ export function resolveFault(config: GatewayConfig, detail: string, toolName: st }; } -async function safeJson(response: Response): Promise<{ error?: Record } | null> { +async function safeJson(response: Response): Promise<{ + error?: Record; + tool_call?: { tool_call_id?: unknown; input?: unknown }; +} | null> { try { - return (await response.json()) as { error?: Record }; + return (await response.json()) as { + error?: Record; + tool_call?: { tool_call_id?: unknown; input?: unknown }; + }; } catch { return null; } diff --git a/integrations/pi/test/argument-transform.test.mjs b/integrations/pi/test/argument-transform.test.mjs new file mode 100644 index 000000000..6b686879b --- /dev/null +++ b/integrations/pi/test/argument-transform.test.mjs @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * The argument-transform decision matrix. + * + * pi validates tool arguments *before* the `tool_call` hook and never + * re-validates, and the extension cannot read the tool's schema, so a rewrite + * that violates the schema would execute. The shape check is what stands in for + * validation: same keys, same JSON types, recursively. These tests pin both the + * cases it must allow and the ones it must refuse. + * + * Run: node --test integrations/pi/test/*.test.mjs + */ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +const { applyTransform, decideTransform, refusalReason, shapeViolation } = await import( + '../src/argument-transform.ts' +); + +const CALL = 'call-1'; +const envelope = (input, id = CALL) => ({ tool_call: { tool_call_id: id, input } }); + +describe('transform decision', () => { + it('has nothing to do when the body carries no transform', () => { + assert.equal(decideTransform({}, CALL, { path: 'a.txt' }).kind, 'none'); + assert.equal(decideTransform(null, CALL, { path: 'a.txt' }).kind, 'none'); + assert.equal(decideTransform({ tool_call: {} }, CALL, { path: 'a.txt' }).kind, 'none'); + }); + + // The use case this exists for: a policy rewriting a path or redacting a value. + it('applies a value rewrite that preserves the shape', () => { + const outcome = decideTransform( + envelope({ path: '.env.example' }), + CALL, + { path: '.env' }, + ); + assert.equal(outcome.kind, 'apply'); + assert.deepEqual(outcome.input, { path: '.env.example' }); + }); + + it('refuses an added or removed key', () => { + const added = decideTransform(envelope({ path: 'a.txt', sudo: true }), CALL, { path: 'a.txt' }); + assert.equal(added.kind, 'refuse'); + assert.match(added.reason, /added sudo/); + + const removed = decideTransform(envelope({}), CALL, { path: 'a.txt' }); + assert.equal(removed.kind, 'refuse'); + assert.match(removed.reason, /removed path/); + }); + + // A required string becoming null is exactly the schema violation pi would execute unchecked. + it('refuses a type change, including to null', () => { + for (const value of [null, 42, ['a.txt'], { nested: true }]) { + const outcome = decideTransform(envelope({ path: value }), CALL, { path: 'a.txt' }); + assert.equal(outcome.kind, 'refuse', JSON.stringify(value)); + assert.match(outcome.reason, /input\.path changed type/); + } + }); + + it('checks nested objects and arrays, naming the path', () => { + const nested = decideTransform( + envelope({ opts: { limit: 'ten' } }), + CALL, + { opts: { limit: 10 } }, + ); + assert.equal(nested.kind, 'refuse'); + assert.match(nested.reason, /input\.opts\.limit changed type from number to string/); + + const shorter = decideTransform(envelope({ paths: ['a'] }), CALL, { paths: ['a', 'b'] }); + assert.equal(shorter.kind, 'refuse'); + assert.match(shorter.reason, /input\.paths changed length from 2 to 1/); + + const ok = decideTransform(envelope({ paths: ['x', 'y'] }), CALL, { paths: ['a', 'b'] }); + assert.equal(ok.kind, 'apply'); + }); + + // A body for a different call means the two sides disagree about what is in flight; applying it + // would rewrite one tool call with another's arguments. + it('refuses a transform addressed to a different tool call', () => { + const outcome = decideTransform(envelope({ path: 'b.txt' }, 'call-2'), CALL, { path: 'a.txt' }); + assert.equal(outcome.kind, 'refuse'); + assert.match(outcome.reason, /call-2, not call-1/); + }); + + it('refuses a non-object transform', () => { + const outcome = decideTransform(envelope('rm -rf /'), CALL, { path: 'a.txt' }); + assert.equal(outcome.kind, 'refuse'); + assert.match(outcome.reason, /is string, not an object/); + }); +}); + +describe('what the shape check does not promise', () => { + // Documented limitation, asserted so it is not mistaken for validation: the extension cannot see + // the tool's schema, so pattern/enum/range violations pass the check and will execute. + it('allows a value the schema might still reject', () => { + assert.equal(shapeViolation({ path: 'a.txt' }, { path: '../../etc/shadow' }), null); + assert.equal(shapeViolation({ mode: 'read' }, { mode: 'not-an-enum-member' }), null); + }); +}); + +describe('applying the transform', () => { + // In place is required, not stylistic: pi hands the same object to the tool and to later + // handlers, so replacing the reference would be silently discarded. + it('mutates the object pi will execute rather than replacing it', () => { + const input = { path: '.env', encoding: 'utf8' }; + const seenByPi = input; + applyTransform(input, { path: '.env.example', encoding: 'utf8' }); + assert.equal(seenByPi.path, '.env.example'); + assert.equal(seenByPi, input); + }); +}); + +describe('the refusal reason', () => { + // pi hands the reason to the model verbatim, so it has to read as guidance and must not look + // like the model did something wrong. + it('names the tool, the cause, and that it is not a judgement of the request', () => { + const reason = refusalReason('read', 'input added sudo'); + assert.match(reason, /read/); + assert.match(reason, /input added sudo/); + assert.match(reason, /not a judgement about your request/); + assert.match(reason, /blocked rather than run with the original/); + }); +}); diff --git a/integrations/pi/test/gateway-client.test.mjs b/integrations/pi/test/gateway-client.test.mjs index 2a80d1399..d512931ee 100644 --- a/integrations/pi/test/gateway-client.test.mjs +++ b/integrations/pi/test/gateway-client.test.mjs @@ -90,7 +90,10 @@ describe('gateway client wire contract', () => { tool_name: 'read', input: { path: 'README.md' }, }); - assert.deepEqual(outcome, { kind: 'allow' }); + // An allow may carry a body (a rewritten payload from a request intercept), so assert the + // verdict rather than the exact object. + assert.equal(outcome.kind, 'allow'); + assert.equal(outcome.reason, undefined); }); it('turns a guardrail 403 into a block carrying the reason verbatim', async () => { From 27373c49e0f7efe9639909a217bd86479bb6c1ba Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 17:52:26 -0700 Subject: [PATCH 12/41] feat(pi): gate pi's bang-prefixed inline shell through the gateway pi's `!cmd` and `!!cmd` never reach the tool registry, so `tool_call` does not fire for them and none of the tool gating covered them. They reach pi's `user_bash` hook instead, which is interceptable: a handler that returns a `BashResult` makes pi skip execution entirely and record that result. The extension now posts the command to `/hooks/pi` as a tool start named `user_bash`, so the same conditional-execution guardrail chain and the same 403 contract decide it. The name is deliberately not `bash`: a guardrail receives only the tool name and the arguments, so a policy can tell a command the user typed from one the model proposed only if the two arrive under different names, and "the model may not run shell commands" should not also stop a human typing `!git status`. The cost is that a policy covering both has to name both, which the docs state. pi gives the hook no block-and-reason contract, so a refusal is a synthetic failed `BashResult` that pi records as though the command had run: exit code 126 (found, but could not be executed), an attribution line, then the guardrail's reason verbatim. `NEMO_RELAY_PI_FAIL` governs this path too. A rewritten command is refused rather than run, because pi's result type can replace the result or the execution backend but never the command itself. `emitUserBash` wraps handlers in try/catch, so a throw here fails open and is invisible -- the opposite of `tool_call`. Every path returns an explicit decision, and the catch re-reads the failure policy rather than defaulting to open, so an explicit fail-closed setting is not overridden by an internal error. Two things beyond the gate itself: - Tool events for a harness that reports its own turn start no longer open a turn when none is open. Inline shell is the first tool event that can arrive between turns -- a command typed at an idle prompt -- and opening a turn to hold it invented a boundary pi never reported. This is the rule `mark` already applies, reached from the tool side. It also changes where a `tool_execution_end` that lands after `turn_end` attaches for pi: on the session scope rather than in a manufactured turn. Codex and Claude Code report no turn start and are unaffected. - The descriptor's `hook_events` gains `tool_arguments_transformed`, which the extension has been posting since argument transforms landed. The list is an inventory of what the extension posts, so a test now pins the exact set. Verified live against pi v0.84.0 driven in RPC mode: an allowed command runs and its span sits directly under the session scope; a command refused by the `examples.rust_native_policy` plugin never executes, and the reason reaches the user verbatim with exit code 126; an unreachable gateway under fail-closed refuses with the infrastructure-fault wording. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- crates/cli/src/agents/pi/adapter.rs | 15 +- crates/cli/src/agents/pi/mod.rs | 7 + crates/cli/src/sessions/mod.rs | 19 +- .../tests/coverage/agents/adapters_tests.rs | 44 +++ .../coverage/agents/coding_agent_tests.rs | 2 +- crates/cli/tests/coverage/agents/pi_tests.rs | 35 +++ .../cli/tests/coverage/shared/server_tests.rs | 121 +++++++ .../tests/coverage/shared/session_tests.rs | 104 ++++++ docs/nemo-relay-cli/pi.mdx | 53 +++- docs/reference/support-matrix.mdx | 2 +- integrations/pi/README.md | 96 +++++- integrations/pi/index.ts | 162 +++++++++- integrations/pi/src/pi-hook-types.ts | 39 +++ integrations/pi/src/user-bash.ts | 120 +++++++ integrations/pi/test/user-bash.test.mjs | 295 ++++++++++++++++++ 15 files changed, 1088 insertions(+), 26 deletions(-) create mode 100644 integrations/pi/src/user-bash.ts create mode 100644 integrations/pi/test/user-bash.test.mjs diff --git a/crates/cli/src/agents/pi/adapter.rs b/crates/cli/src/agents/pi/adapter.rs index 0e50e5b37..048c3ee9f 100644 --- a/crates/cli/src/agents/pi/adapter.rs +++ b/crates/cli/src/agents/pi/adapter.rs @@ -46,8 +46,19 @@ pub(crate) fn adapt(payload: Value, headers: &HeaderMap) -> AdapterOutcome { // subagents are an extension-level concept it does not surface. subagent_start: &[], subagent_end: &[], - tool_start: &["tool_call", "toolCall"], - tool_end: &["tool_execution_end", "toolExecutionEnd"], + // `user_bash` is pi's bang-prefixed inline shell, which never + // reaches the tool registry and so never fires `tool_call`. It is + // gated as a tool start under its own tool name so the same + // conditional-execution guardrail chain decides it; `user_bash_end` + // is synthesized by the extension, because pi reports no completion + // for inline shell. + tool_start: &["tool_call", "toolCall", "user_bash", "userBash"], + tool_end: &[ + "tool_execution_end", + "toolExecutionEnd", + "user_bash_end", + "userBashEnd", + ], // pi has an explicit turn boundary at both ends, unlike Codex and // Claude Code which only signal it through `Stop`. Classifying the // open as well as the close is what stops the gateway inventing a diff --git a/crates/cli/src/agents/pi/mod.rs b/crates/cli/src/agents/pi/mod.rs index f0785ca0b..509a21bd2 100644 --- a/crates/cli/src/agents/pi/mod.rs +++ b/crates/cli/src/agents/pi/mod.rs @@ -44,6 +44,13 @@ pub(super) const DESCRIPTOR: AgentDescriptor = AgentDescriptor { "turn_end", "tool_call", "tool_execution_end", + // Not a pi hook name: posted after a request intercept's rewrite is applied to a tool + // call, so the trace records that the arguments the tool ran were not the ones proposed. + "tool_arguments_transformed", + // The bang-prefixed inline shell gate. `user_bash` is a pi hook; `user_bash_end` is not -- + // pi reports no completion for inline shell, so the extension synthesizes the close. + "user_bash", + "user_bash_end", // Not a pi hook name: the extension posts this after deciding whether to point the active // model's provider at the gateway, so a trace with no LLM spans carries its own reason. "model_redirect", diff --git a/crates/cli/src/sessions/mod.rs b/crates/cli/src/sessions/mod.rs index 1536cc379..5bd95ba0b 100644 --- a/crates/cli/src/sessions/mod.rs +++ b/crates/cli/src/sessions/mod.rs @@ -1096,6 +1096,21 @@ impl Session { self.open_turn(event_metadata, Value::Null, "implicit") } + // Chooses the enclosing scope for a tool event that arrives with no turn open. + // + // Same reasoning as `mark`: a harness that reports its own turn start is telling the gateway + // where turns begin, so a tool event between turns is genuinely between turns, and opening one + // to hold it invents a turn the harness never reported. pi's inline-shell gate is what makes + // this reachable -- a user typing `!cmd` at an idle prompt is outside every turn, while pi's + // model-invoked tool calls always arrive inside one. Codex and Claude Code report no turn + // start, so they keep opening turns lazily and are unaffected. + fn ensure_tool_scope_started(&mut self, event_metadata: Value) -> Result<(), CliError> { + if self.agent_kind.has_explicit_turn_start() && self.turn_scope.is_none() { + return self.ensure_agent_started(event_metadata); + } + self.ensure_turn_started(event_metadata) + } + fn ensure_turn_started_for_gateway(&mut self, start: &LlmGatewayStart) -> Result<(), CliError> { if self.turn_scope.is_some() { return Ok(()); @@ -1530,7 +1545,7 @@ impl Session { // scope. Duplicate tool IDs are ignored so repeated pre-tool hooks do not create parallel // handles for one agent tool invocation. async fn start_tool(&mut self, event: ToolEvent) -> Result<(), CliError> { - self.ensure_turn_started(event.metadata.clone())?; + self.ensure_tool_scope_started(event.metadata.clone())?; if self.tools.contains_key(&event.tool_call_id) { return Ok(()); } @@ -1600,7 +1615,7 @@ impl Session { // Ends a tool call, synthesizing a start if no matching handle exists. This keeps post-only // hooks observable and preserves the final result/status instead of dropping orphaned endings. async fn end_tool(&mut self, event: ToolEvent) -> Result, CliError> { - self.ensure_turn_started(event.metadata.clone())?; + self.ensure_tool_scope_started(event.metadata.clone())?; let event_metadata = self.event_identity_metadata(event.metadata.clone()); let completed_agent_subagent_id = alignment::completed_subagent_from_tool(&event); let explicit_subagent_id = event diff --git a/crates/cli/tests/coverage/agents/adapters_tests.rs b/crates/cli/tests/coverage/agents/adapters_tests.rs index 0ddc31abc..4738a5503 100644 --- a/crates/cli/tests/coverage/agents/adapters_tests.rs +++ b/crates/cli/tests/coverage/agents/adapters_tests.rs @@ -1159,6 +1159,50 @@ fn pi_compaction_is_classified_only_once_it_has_happened() { ); } +// pi's bang-prefixed inline shell never reaches the tool registry, so it never fires `tool_call` +// and none of the tool gating covers it. It is classified as a tool boundary so the same +// conditional-execution guardrail chain decides it; the close is synthesized by the extension, +// because pi reports no completion for inline shell. +#[test] +fn pi_inline_shell_is_classified_as_a_tool_boundary() { + let started = pi::adapt( + json!({ + "session_id": "pi-session", + "hook_event_name": "user_bash", + "tool_call_id": "user-bash-0", + "tool_name": "user_bash", + "input": { "command": "git status", "cwd": "/work", "exclude_from_context": false } + }), + &HeaderMap::new(), + ); + match started.events.as_slice() { + [NormalizedEvent::ToolStarted(event)] => { + // The name a guardrail matches on. Deliberately not `bash`: a policy has to be able to + // tell a command the user typed from one the model proposed, and the guardrail chain + // sees only the name and the arguments. + assert_eq!(event.tool_name, "user_bash"); + assert_eq!(event.arguments["command"], json!("git status")); + } + events => panic!("user_bash must open a tool span. events: {events:?}"), + } + + let ended = pi::adapt( + json!({ + "session_id": "pi-session", + "hook_event_name": "user_bash_end", + "tool_call_id": "user-bash-0", + "tool_name": "user_bash", + "status": "error" + }), + &HeaderMap::new(), + ); + assert!( + matches!(ended.events.as_slice(), [NormalizedEvent::ToolEnded(_)]), + "user_bash_end must close the tool span. events: {:?}", + ended.events + ); +} + // The promotion is what makes attribution survive on tool events at all: the session manager // builds tool spans from the extracted call id, name, arguments, result and metadata, and drops // the raw payload. Without this the keys would be accepted on the wire and silently discarded. diff --git a/crates/cli/tests/coverage/agents/coding_agent_tests.rs b/crates/cli/tests/coverage/agents/coding_agent_tests.rs index 71698eba3..e55a97188 100644 --- a/crates/cli/tests/coverage/agents/coding_agent_tests.rs +++ b/crates/cli/tests/coverage/agents/coding_agent_tests.rs @@ -22,7 +22,7 @@ fn agent_descriptors_are_complete_and_unique() { assert_eq!(CodingAgent::ClaudeCode.hook_events().len(), 14); assert_eq!(CodingAgent::Codex.hook_events().len(), 10); assert_eq!(CodingAgent::Pi.label(), "pi"); - assert_eq!(CodingAgent::Pi.hook_events().len(), 12); + assert_eq!(CodingAgent::Pi.hook_events().len(), 15); for agent in CodingAgent::ALL { let events = agent.hook_events(); assert!(events.iter().all(|event| !event.is_empty())); diff --git a/crates/cli/tests/coverage/agents/pi_tests.rs b/crates/cli/tests/coverage/agents/pi_tests.rs index c09e4e1ac..c2d742c92 100644 --- a/crates/cli/tests/coverage/agents/pi_tests.rs +++ b/crates/cli/tests/coverage/agents/pi_tests.rs @@ -31,4 +31,39 @@ fn hook_events_use_pi_vocabulary_not_codex_vocabulary() { assert!(DESCRIPTOR.hook_events.contains(&"tool_call")); assert!(DESCRIPTOR.hook_events.contains(&"agent_settled")); assert!(!DESCRIPTOR.hook_events.contains(&"PreToolUse")); + // Never forwarded: it fires before validation and for calls that never execute. + assert!(!DESCRIPTOR.hook_events.contains(&"tool_execution_start")); +} + +// The list gates nothing inbound -- an unrecognized event name becomes a mark rather than an +// error -- so its whole value is being an accurate inventory of what the extension posts. Pinning +// the exact set is what makes a hook added on one side and forgotten on the other visible here +// rather than in a trace nobody reads. +#[test] +fn hook_events_inventory_matches_what_the_extension_posts() { + let mut declared = DESCRIPTOR.hook_events.to_vec(); + declared.sort_unstable(); + assert_eq!( + declared, + [ + "agent_end", + "agent_settled", + "agent_start", + // Not pi hook names. The extension synthesizes these three: two report a decision it + // took (redirect the model's provider, apply a rewrite), and `user_bash_end` closes + // the inline-shell span because pi reports no completion for it. + "model_redirect", + "session_before_compact", + "session_compact", + "session_shutdown", + "session_start", + "tool_arguments_transformed", + "tool_call", + "tool_execution_end", + "turn_end", + "turn_start", + "user_bash", + "user_bash_end", + ] + ); } diff --git a/crates/cli/tests/coverage/shared/server_tests.rs b/crates/cli/tests/coverage/shared/server_tests.rs index 923e6b2f8..abf765339 100644 --- a/crates/cli/tests/coverage/shared/server_tests.rs +++ b/crates/cli/tests/coverage/shared/server_tests.rs @@ -2375,6 +2375,127 @@ async fn pi_tool_call_hook_allows_when_no_guardrail_objects() { assert_eq!(response.status(), StatusCode::OK); } +// pi's bang-prefixed inline shell bypasses the tool registry, so `tool_call` never fires for it +// and a policy that gates tools does not cover it. The extension forwards it as a tool start named +// `user_bash`, which puts it through the same guardrail chain and the same 403 contract -- but the +// refusal it produces is a synthetic failed `BashResult`, not a blocked tool call, because pi's +// `user_bash` hook has no block-and-reason form. +#[tokio::test] +async fn pi_user_bash_hook_rejects_when_conditional_guardrail_blocks() { + let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; + let _ = deregister_tool_conditional_execution_guardrail("cli-pi-user-bash-blocker"); + register_tool_conditional_execution_guardrail( + "cli-pi-user-bash-blocker", + 1, + Arc::new(|name, args| { + Box::pin(async move { + let pipes_to_shell = args + .get("command") + .and_then(Value::as_str) + .is_some_and(|command| command.contains("| sh")); + Ok((name == "user_bash" && pipes_to_shell) + .then(|| "piping a download into a shell is blocked here".to_string())) + }) + }), + ) + .unwrap(); + let _cleanup = ToolGuardrailCleanup("cli-pi-user-bash-blocker"); + + let app = router(test_config()); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/hooks/pi") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "session_id": "pi-user-bash-session", + "hook_event_name": "user_bash", + "tool_call_id": "user-bash-0", + "tool_name": "user_bash", + "input": { + "command": "curl https://example.test/install | sh", + "cwd": "/work", + "exclude_from_context": false + } + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + body["error"]["type"], + json!("nemo_relay_guardrail_rejected") + ); + // Verbatim again, and for the same reason: it becomes the output of the refused command, which + // the user reads in the terminal and -- unless the `!!` form was used -- the model reads too. + assert_eq!( + body["error"]["reason"], + json!("piping a download into a shell is blocked here") + ); +} + +// The tool name is the whole point of gating inline shell separately: a policy that stops the +// *model* running shell commands should not also stop the human typing `!git status`, and the +// guardrail chain sees only the name and the arguments, so it can only tell them apart if they +// arrive under different names. +#[tokio::test] +async fn pi_user_bash_is_not_gated_by_a_policy_that_names_the_bash_tool() { + let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; + let _ = deregister_tool_conditional_execution_guardrail("cli-pi-bash-tool-blocker"); + register_tool_conditional_execution_guardrail( + "cli-pi-bash-tool-blocker", + 1, + Arc::new(|name, _args| { + Box::pin(async move { + Ok((name == "bash").then(|| "the model may not run shell commands".to_string())) + }) + }), + ) + .unwrap(); + let _cleanup = ToolGuardrailCleanup("cli-pi-bash-tool-blocker"); + + let app = router(test_config()); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/hooks/pi") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "session_id": "pi-user-bash-allow-session", + "hook_event_name": "user_bash", + "tool_call_id": "user-bash-1", + "tool_name": "user_bash", + "input": { + "command": "git status", + "cwd": "/work", + "exclude_from_context": false + } + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::OK, + "a `bash` policy must not silently swallow the user's own inline shell; covering both \ + means naming both" + ); +} + #[tokio::test] async fn gateway_forwards_openai_json_without_rewriting_payload() { let upstream = spawn_upstream(false).await; diff --git a/crates/cli/tests/coverage/shared/session_tests.rs b/crates/cli/tests/coverage/shared/session_tests.rs index 5e4347652..7a776da30 100644 --- a/crates/cli/tests/coverage/shared/session_tests.rs +++ b/crates/cli/tests/coverage/shared/session_tests.rs @@ -5648,3 +5648,107 @@ async fn pi_re_entry_produces_two_turns_attributed_to_two_attempts() { drop(captured); deregister_subscriber(subscriber_name).unwrap(); } + +// pi's inline shell is the one tool event that can arrive with no turn open: `!git status` typed +// at an idle prompt happens between turns, not inside one. Opening a turn to hold it would invent +// a boundary pi never reported -- the same defect `has_explicit_turn_start` already fixed for +// marks, reached from the tool side instead. +#[tokio::test] +async fn pi_inline_shell_between_turns_does_not_invent_a_turn() { + let subscriber_name = "cli-pi-inline-shell-scope-test"; + let _ = deregister_subscriber(subscriber_name); + let captured = Arc::new(StdMutex::new(Vec::<(String, Option)>::new())); + let events = captured.clone(); + register_subscriber( + subscriber_name, + Arc::new(move |event| { + let Some(metadata) = event.metadata() else { + return; + }; + if metadata.get("session_id").and_then(Value::as_str) != Some("pi-inline-shell-session") + { + return; + } + events + .lock() + .unwrap() + .push((event.name().to_string(), event.scope_category())); + }), + ) + .unwrap(); + + let manager = SessionManager::new(session_test_config()); + let session = json!({ "session_id": "pi-inline-shell-session" }); + for payload in [ + json!({ "hook_event_name": "session_start", "reason": "startup" }), + json!({ "hook_event_name": "agent_start", "attempt_index": 0 }), + json!({ + "hook_event_name": "turn_start", "turn_index": 0, "turn_seq": 0, "attempt_index": 0 + }), + json!({ + "hook_event_name": "turn_end", "turn_index": 0, "turn_seq": 0, "attempt_index": 0 + }), + // The user types `!git status` at the prompt, after the turn closed. + json!({ + "hook_event_name": "user_bash", "tool_call_id": "user-bash-0", + "tool_name": "user_bash", + "input": { "command": "git status", "cwd": "/work", "exclude_from_context": false }, + "attempt_index": 0, "turn_seq": 0 + }), + json!({ + "hook_event_name": "user_bash_end", "tool_call_id": "user-bash-0", + "tool_name": "user_bash", "status": "ok", "attempt_index": 0, "turn_seq": 0 + }), + json!({ "hook_event_name": "agent_end", "attempt_index": 0 }), + json!({ "hook_event_name": "agent_settled", "attempts": 1, "attempt_index": 0 }), + json!({ "hook_event_name": "session_shutdown", "reason": "quit" }), + ] { + let mut merged = session.clone(); + merged + .as_object_mut() + .unwrap() + .extend(payload.as_object().unwrap().clone()); + apply_pi_hook(&manager, merged).await; + } + + flush_subscribers().unwrap(); + let captured = captured.lock().unwrap(); + let sequence = captured + .iter() + .map(|(name, category)| (name.as_str(), *category)) + .collect::>(); + + assert_eq!( + sequence + .iter() + .filter(|(name, category)| *name == "pi-turn" && *category == Some(ScopeCategory::Start)) + .count(), + 1, + "pi reported one turn; the inline shell command must not open a second. sequence: \ + {sequence:?}" + ); + // The span still exists -- it is not being dropped, only re-parented onto the session scope, + // which is where a command typed between turns belongs. + assert!( + sequence.contains(&("user_bash", Some(ScopeCategory::Start))) + && sequence.contains(&("user_bash", Some(ScopeCategory::End))), + "the inline shell command must still produce a balanced span. sequence: {sequence:?}" + ); + let turn_end = sequence + .iter() + .position(|(name, category)| *name == "pi-turn" && *category == Some(ScopeCategory::End)) + .expect("the one turn should close"); + let shell_start = sequence + .iter() + .position(|(name, category)| { + *name == "user_bash" && *category == Some(ScopeCategory::Start) + }) + .expect("the inline shell span should open"); + assert!( + turn_end < shell_start, + "the command was typed after the turn closed, so its span belongs outside it. sequence: \ + {sequence:?}" + ); + drop(captured); + deregister_subscriber(subscriber_name).unwrap(); +} diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index 3f677c294..87b88799d 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -109,10 +109,11 @@ it can block indefinitely before any hook fires. ## Tool Gating -`tool_call` is the only pi hook that can block, and for model-invoked tools it -is the only pre-execution decision point that sees arguments: pi applies -`--tools`, `--exclude-tools`, `--no-tools`, and the runtime `setActiveTools` at -tool-registry construction, never per call. +For model-invoked tools, `tool_call` is the only pre-execution decision point +that sees arguments: pi applies `--tools`, `--exclude-tools`, `--no-tools`, and +the runtime `setActiveTools` at tool-registry construction, never per call. The +user's own inline shell takes a different path and is gated separately. See +[Inline Shell Gating](#inline-shell-gating). A guardrail rejection surfaces as HTTP 403 with `error.type = "nemo_relay_guardrail_rejected"` and the guardrail's own words in @@ -162,9 +163,48 @@ A transform that violates the invariant **blocks the call**, with a reason stating that a policy could not be applied rather than that the request was refused. Running the original arguments would silently discard the policy. +## Inline Shell Gating + +pi's bang prefix runs a command outside the tool registry: `!git status` runs it +and shows the model the output, and `!!git status` runs it and keeps the output +out of the model's context. Neither fires `tool_call`, so tool gating does not +cover them. Both reach pi's `user_bash` hook, which the extension gates through +the same guardrail chain by posting the command as a tool span start. + + +The tool name is `user_bash`, not `bash`. A guardrail receives only the tool name +and the arguments, so a policy can distinguish a command the user typed from one +the model proposed only if the two arrive under different names. **A policy that +must cover both has to name both** — a rule written for `bash` alone does not +gate the bang prefix. + + +The posted arguments are `command`, `cwd`, and `exclude_from_context`, the last +of which is `true` for the `!!` form. + +pi's `user_bash` hook has no block-and-reason contract, so a refusal is a +synthetic failed command result that pi records as though the command had run: + +| Field | Value | +| --- | --- | +| `exitCode` | `126`, the shell convention for a command that was found but could not be executed | +| `output` | An attribution line, a blank line, then the guardrail's reason verbatim | +| `cancelled` | `false` — nothing was started | +| `truncated` | `false` — the message is whole | + +`NEMO_RELAY_PI_FAIL` governs this path as well. A rewritten command is refused +rather than run: pi's `user_bash` result can replace the result or the execution +backend, but never the command itself. + +Two limits are worth planning around. The gate records the decision, not the +command — pi reports no completion for inline shell, so the span closes as soon +as the verdict arrives and measures the policy round trip. And the hook fires +only in the interactive TUI and in RPC mode; headless `-p` has no input loop to +type a bang prefix into. + ## Captured Events -The extension posts 11 hooks. pi reports both ends of a turn, so Relay opens the +The extension posts 15 hooks. pi reports both ends of a turn, so Relay opens the turn scope at pi's own `turn_start` instead of inferring one, and a mark that arrives between turns is recorded on the session scope rather than opening an empty turn to hold it. @@ -177,8 +217,11 @@ empty turn to hold it. | `session_before_compact` | Mark. The compaction is announced, not yet done, and a later extension can still cancel it | | `session_compact` | Canonical `compaction` mark | | `model_select` | `model_redirect` mark recording whether redirection applied | +| *(after a rewrite)* | `tool_arguments_transformed` mark, so the trace records that the arguments the tool ran were not the ones proposed | | `tool_call` | Tool span start, and the gate | | `tool_execution_end` | Tool span end, for every outcome including blocked | +| `user_bash` | Tool span start named `user_bash`, and the inline shell gate | +| `user_bash_end` | Tool span end. Synthesized by the extension, because pi reports no completion for inline shell | pi's `tool_execution_start` is deliberately not forwarded: it fires before argument validation and also for calls that never execute. `tool_result` is not diff --git a/docs/reference/support-matrix.mdx b/docs/reference/support-matrix.mdx index d61821b3a..62b71f49d 100644 --- a/docs/reference/support-matrix.mdx +++ b/docs/reference/support-matrix.mdx @@ -66,7 +66,7 @@ and older CLI versions during installation, diagnostics, and transparent runs. | --- | --- | --- | --- | | Claude Code | 2.1.121 | Persistent install, transparent run, lifecycle hooks, local gateway routing, and pre-tool security | Claude desktop, web, and application sessions are unsupported unless they expose the same local hook and gateway controls. Optimization requires gateway-routed LLM traffic and available hooks. | | Codex CLI | 0.143.0 | Persistent install, transparent run, 10 supported plugin hooks, local gateway routing, and pre-tool security | Cloud or remote tasks that bypass the local machine have partial or no LLM capture. The plugin hook schema has no `SessionEnd`; Relay finalizes the cumulative session snapshot at `Stop`. Encrypted Codex multi-agent v2 payloads cannot be decrypted or reliably linked. | -| pi | 0.84.0 | Transparent run through a Relay-authored pi extension, 11 lifecycle hooks, tool-call security, and model-call security when redirection applies | Proof of concept. No persistent install: pi has no plugin marketplace. Model traffic is redirected only when the gateway forwards to the endpoint the selected model would otherwise call; otherwise there are no LLM spans for that model. Six of pi's 38 providers speak an API the gateway has no route for. Subagents and nested pi processes appear as unrelated sessions. pi ships breaking changes through minor releases, so hook signatures need re-verification after an upgrade. | +| pi | 0.84.0 | Transparent run through a Relay-authored pi extension, 13 lifecycle hooks, tool-call security, inline-shell security, and model-call security when redirection applies | Proof of concept. No persistent install: pi has no plugin marketplace. Model traffic is redirected only when the gateway forwards to the endpoint the selected model would otherwise call; otherwise there are no LLM spans for that model. Six of pi's 38 providers speak an API the gateway has no route for. Subagents and nested pi processes appear as unrelated sessions. pi ships breaking changes through minor releases, so hook signatures need re-verification after an upgrade. | For installation, diagnostics, and host-specific behavior, refer to [Coding Agent Installation](/nemo-relay-cli/plugin-installation), [Claude diff --git a/integrations/pi/README.md b/integrations/pi/README.md index fb90c8b2d..908addbc7 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -48,17 +48,18 @@ directory (`~/.pi/agent/extensions/`, `.pi/extensions/`). |---|---|---| | `NEMO_RELAY_PI_GATEWAY_URL` | `http://127.0.0.1:4040` | Gateway base URL | | `NEMO_RELAY_PI_TIMEOUT_MS` | `5000` | Per-request timeout | -| `NEMO_RELAY_PI_FAIL` | `open` | `closed` blocks tool calls when the gateway is unreachable | +| `NEMO_RELAY_PI_FAIL` | `open` | `closed` blocks tool calls and inline shell commands when the gateway is unreachable | | `NEMO_RELAY_PI_REDIRECT` | `match` | `force` redirects without checking the upstream; `off` disables redirection | | `NEMO_RELAY_PI_OPENAI_UPSTREAM` | unset | What the gateway forwards OpenAI-compatible traffic to. Set by the launcher | | `NEMO_RELAY_PI_ANTHROPIC_UPSTREAM` | unset | What the gateway forwards Anthropic traffic to. Set by the launcher | ## How tool gating works -`tool_call` is the only pi hook that can block, and for model-invoked tools it -is the only pre-execution decision point that sees arguments — pi's `--tools`, -`--exclude-tools`, `--no-tools` and runtime `setActiveTools` are all applied at -tool-registry construction, never per call. +For model-invoked tools, `tool_call` is the only pre-execution decision point +that sees arguments — pi's `--tools`, `--exclude-tools`, `--no-tools` and +runtime `setActiveTools` are all applied at tool-registry construction, never +per call. The user's own inline shell takes a different path and is gated +separately; see [Inline shell](#inline-shell). The wire contract, pinned from both sides by tests: @@ -112,6 +113,73 @@ the transform existed to prevent. This is a different axis from `NEMO_RELAY_PI_FAIL`, which governs an unreachable gateway rather than one that answered with something unusable. +## Inline shell + +pi's bang prefix runs a command without going through the tool registry: +`!git status` runs it and shows the model the output, `!!git status` runs it and +keeps the output out of the model's context. Neither fires `tool_call`, so none +of the gating above sees them. They reach `user_bash` instead, which this +extension gates the same way — the command is posted to the gateway as a tool +start and the same conditional-execution guardrail chain decides it. + +**The tool name is `user_bash`, not `bash`.** A guardrail receives only the tool +name and the arguments, so if both arrived as `bash` a policy could not tell a +command the user typed from one the model proposed — and "the model may not run +shell commands" is a reasonable rule that should not also stop a human typing +`!git status`. The cost is that **a policy which wants to cover both has to name +both**; a rule written only for `bash` does not gate the bang prefix. + +| Argument | Value | +|---|---| +| `command` | The command text, exactly as typed after the prefix | +| `cwd` | pi's working directory for the command | +| `exclude_from_context` | `true` for the `!!` form, whose output the model never sees | + +### The refusal shape + +pi gives `user_bash` no block-and-reason contract — there is no `{block, +reason}` here — so a refusal has to be a **synthetic failed command result**, +which pi records exactly as if the command had run: + +| Field | Value | Why | +|---|---|---| +| `exitCode` | `126` | The shell convention for "found, but could not be executed". Not `0` (reads as success), not `1` (indistinguishable from the command itself failing), not `127` (sends you hunting for a missing binary) | +| `output` | `NeMo Relay blocked this command.` then a blank line, then the reason verbatim | The attribution line says what declined it; the reason is the guardrail's own words, and for `!cmd` the model reads them | +| `cancelled` | `false` | Nothing was started, so nothing was interrupted | +| `truncated` | `false` | The message is whole | + +`NEMO_RELAY_PI_FAIL` governs this path too: a gateway that cannot be reached +allows the command by default, and refuses it under `closed` with a reason that +says explicitly that it is an infrastructure fault rather than a judgement. + +**A rewritten command is refused, not run.** pi's `user_bash` result can replace +the *result* or the execution backend, but never the command — both call sites +pass the original text on, and the terminal component has already been built +from it. So a request intercept that rewrites an inline command cannot be +honoured, and the command is refused rather than run unmodified, on the same +rule the tool path applies to a transform it cannot apply safely. + +### Limits worth knowing + +- **The gate decides; it does not observe.** pi has no completion hook for + inline shell, so on an allow the span closes immediately: it measures the + policy round trip, not the command. Taking execution over to fix that would + mean handing pi custom `operations`, which drops the user's configured shell + path and pi's process-tree cancellation — the extension does not change how pi + runs things. +- **`!!` hides the refusal from the model**, not from the user. The terminal + shows it either way; the model's context does not. +- **The first extension to answer wins**, and there is no priority system. Loaded + with `pi -e` this extension answers first; installed with `pi install` it loads + last, and an extension ahead of it that answers `user_bash` unconditionally — + pi's own `sandbox` and `gondolin` examples do — means the gateway never sees the + command. +- **Where it fires.** The interactive TUI and RPC mode. Headless `-p` has no + input loop to type a bang prefix into, so there is nothing to gate there. +- **The prompt looks frozen while the gate is out.** pi builds the terminal + component *after* the hook resolves, so a slow gateway shows nothing at all + until `NEMO_RELAY_PI_TIMEOUT_MS` expires. Keep that timeout short. + ## Model redirection pi resolves a base URL per model from a generated catalog, so there is no flag or @@ -198,11 +266,14 @@ per-call state is keyed by `toolCallId`, the only correlator pi provides. | `turn_start` | turn scope **open** | Carries `turn_index`, `turn_seq`, `attempt_index`. Awaited, so the turn exists before pi's model call arrives | | `turn_end` | turn scope **close** | Carries `turn_index`, `turn_seq`, `attempt_index`. Awaited, for the same reason | | `model_select` | `model_redirect` mark | Re-evaluates redirection for the newly selected model | +| *(after a rewrite)* | `tool_arguments_transformed` mark | Synthesized, so the trace records that the arguments the tool ran were not the ones proposed | | `session_before_compact` | mark | Announced, not done, and cancellable by a later extension. Carries `reason`, `will_retry`, `tokens_before` | | `session_compact` | compaction | The completed compaction, which the runtime treats as proof the context was rebuilt | | `tool_call` | tool start, and the gate | The only blocking hook. Carries `attempt_index`, `turn_seq` | | `tool_execution_end` | tool end | For **every** outcome, including blocked. Carries `attempt_index`, `turn_seq` | | `tool_execution_start` | *not forwarded* | Registered, but only to remember a tool name for the matching end: it fires before validation and for calls that never execute | +| `user_bash` | tool start, and the second gate | The bang prefix, which never reaches the tool registry. Gated under the tool name `user_bash` — see [Inline shell](#inline-shell) | +| *(synthesized)* `user_bash_end` | tool end | pi reports no completion for inline shell, so the extension closes the span itself | `tool_result` is deliberately unused: it does not fire for blocked calls, and in the parallel path it fires *before* `tool_execution_end`. @@ -228,7 +299,14 @@ worth stating separately: a child pi process running this extension resolves its *own* session id and posts under it, so it does not appear as a subagent of the parent. It appears as an unrelated session. -**LLM spans**, until model redirection lands. See [Status](#status). +**LLM spans, when redirection is skipped.** They are present whenever the +gateway fronts the endpoint the active model would otherwise have called, and +absent otherwise — the `model_redirect` mark in the trace names which it was and +why. See [Model redirection](#model-redirection). + +**The outcome of an inline shell command.** pi reports no completion for the bang +prefix, so the gate records the decision, not the command. See +[Inline shell](#inline-shell). ## Development @@ -238,8 +316,10 @@ node --test integrations/pi/test/*.test.mjs ``` The gateway half of the contract is covered in Rust by -`pi_tool_call_hook_rejects_when_conditional_guardrail_blocks` and -`pi_tool_call_hook_allows_when_no_guardrail_objects` in +`pi_tool_call_hook_rejects_when_conditional_guardrail_blocks`, +`pi_tool_call_hook_allows_when_no_guardrail_objects`, +`pi_user_bash_hook_rejects_when_conditional_guardrail_blocks` and +`pi_user_bash_is_not_gated_by_a_policy_that_names_the_bash_tool` in `crates/cli/tests/coverage/shared/server_tests.rs`. `test/fixtures/reentry-driver.ts` forces exactly one agent-run re-entry through diff --git a/integrations/pi/index.ts b/integrations/pi/index.ts index b0ea24233..94dc0b4ce 100644 --- a/integrations/pi/index.ts +++ b/integrations/pi/index.ts @@ -9,13 +9,18 @@ * is a thin HTTP client to the NeMo Relay CLI gateway: it forwards pi's * lifecycle to `/hooks/pi`, and gates tool calls on the gateway's verdict. * - * **Governance.** `tool_call` is the only pi hook that can block, and for - * model-invoked tools it is the only pre-execution decision point that sees - * arguments -- `--tools` / `--exclude-tools` / `--no-tools` and the runtime - * `setActiveTools` are applied at tool-registry construction, never per call. - * A gateway guardrail rejection arrives as HTTP 403 and is translated into - * `{block, reason}`; pi hands that reason to the model verbatim, so the model - * reads the guardrail's own words. + * **Governance, on two paths.** For model-invoked tools, `tool_call` is the only + * pre-execution decision point that sees arguments -- `--tools` / + * `--exclude-tools` / `--no-tools` and the runtime `setActiveTools` are applied + * at tool-registry construction, never per call. A gateway guardrail rejection + * arrives as HTTP 403 and is translated into `{block, reason}`; pi hands that + * reason to the model verbatim, so the model reads the guardrail's own words. + * + * The user's own bang-prefixed shell (`!cmd`) never reaches the tool registry, + * so `tool_call` cannot see it. It is gated separately through `user_bash`, + * under its own tool name and with a refusal shaped as a failed command rather + * than as a blocked tool call, because pi gives that hook no reason contract. + * See `src/user-bash.ts`. * * **Lifecycle mapping.** Two pi shapes make a naive mapping wrong: * @@ -68,6 +73,13 @@ import { isNotable, redirectConfigFromEnv, } from './src/provider-redirect.ts'; +import { + USER_BASH_END_HOOK, + USER_BASH_HOOK, + USER_BASH_TOOL_NAME, + refusalResult, + transformRefusalReason, +} from './src/user-bash.ts'; import type { AgentEndEvent, AgentSettledEvent, @@ -85,6 +97,8 @@ import type { ToolExecutionStartEvent, TurnEndEvent, TurnStartEvent, + UserBashEvent, + UserBashEventResult, } from './src/pi-hook-types.ts'; export default function nemoRelayExtension(pi: ExtensionAPI): void { @@ -96,6 +110,15 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { let turnSeq = 0; /** Tool names by call id, so the end payload can name the tool pi started. */ const toolNames = new Map(); + /** + * Inline shell commands have no pi-supplied identifier. + * + * `user_bash` carries only the command text, so the gate synthesizes a call + * id to correlate its own start and end. Per runtime rather than globally + * unique, which is all the gateway's tool map needs: it is keyed by call id + * within one session. + */ + let userBashSeq = 0; /** * Serializes every post to the gateway, in hook order. @@ -469,6 +492,131 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { ...attribution(), }); }); + + // --------------------------------------------------------------------------- + // Inline shell + // --------------------------------------------------------------------------- + + /** Close the gate span, whatever the outcome, so the trace stays balanced. */ + const endUserBash = ( + ctx: ExtensionContext, + callId: string, + status: 'ok' | 'error', + content: string, + ): void => { + emit(ctx, { + hook_event_name: USER_BASH_END_HOOK, + tool_call_id: callId, + tool_name: USER_BASH_TOOL_NAME, + result: { content }, + status, + ...attribution(), + }); + }; + + /** + * The second governance seam: pi's bang-prefixed inline shell. + * + * `!cmd` and `!!cmd` bypass the tool registry entirely -- they reach + * `emitUserBash`, not `tool_call` -- so nothing about the tool gate covers + * them. This closes that gap using the gateway machinery that already exists: + * the command is posted as a tool start named `user_bash`, so the same + * conditional-execution guardrail chain decides it and the same 403 contract + * carries the reason back. + * + * Three pi facts shape every line of this handler. + * + * 1. **There is no block-and-reason contract.** A refusal is a synthetic + * failed `BashResult` and pi records it exactly as if the command had run. + * See `src/user-bash.ts` for the shape and why those field values. + * 2. **`emitUserBash` catches**, so throwing fails *open* and the command + * runs unchecked. Nothing here is allowed to escape; the catch resolves an + * internal failure through the same policy as an unreachable gateway. + * 3. **The first handler to return anything wins.** Loaded with `-e` this one + * is first; installed with `pi install` it loads last and an earlier + * extension can preempt it, in which case the gateway never sees the + * command at all. + * + * What the gate does *not* do is observe the command. pi has no completion + * hook for inline shell -- `user_bash` is the only one -- so on an allow the + * span closes immediately and measures the policy round trip, not the + * command. Taking execution over to fix that would mean supplying pi with + * custom `operations`, which drops the user's configured shell path and pi's + * process-tree cancellation; the sidecar does not change how pi runs things. + */ + pi.on( + 'user_bash', + async ( + event: UserBashEvent, + ctx: ExtensionContext, + ): Promise => { + const callId = `user-bash-${userBashSeq++}`; + try { + const active = ensureConfig(ctx); + // Enqueued like `tool_call`, so a turn boundary posted earlier in the + // same turn reaches the gateway first. + const outcome = await enqueue(() => + postHook(active, { + hook_event_name: USER_BASH_HOOK, + tool_call_id: callId, + tool_name: USER_BASH_TOOL_NAME, + input: { + command: event.command, + cwd: event.cwd, + // `!!` keeps the command and its output out of the model's + // context, including the output of a refusal. A policy may + // reasonably care. + exclude_from_context: event.excludeFromContext, + }, + ...attribution(), + }), + ); + + const decision = + outcome.kind === 'fault' + ? resolveFault(active, outcome.detail, USER_BASH_TOOL_NAME) + : outcome; + + if (decision.kind === 'block') { + endUserBash(ctx, callId, 'error', decision.reason); + return { result: refusalResult(decision.reason) }; + } + + // An intercept rewrote the command. pi's result type can replace the + // result or the execution backend but never the command itself, so the + // rewrite cannot be honoured -- and running the original would discard + // the policy decision. Refuse, and say which of the two it is. + if (decision.kind === 'allow' && decision.body?.tool_call?.input !== undefined) { + const reason = transformRefusalReason(); + endUserBash(ctx, callId, 'error', reason); + return { result: refusalResult(reason) }; + } + + endUserBash(ctx, callId, 'ok', 'Allowed by policy; pi executed the command.'); + // `undefined` is the only correct allow value: any object at all is a + // result or an operations override, and either would stop pi running + // the command as the user typed it. + return undefined; + } catch (error) { + // Reached only if something inside this handler failed -- `postHook` + // resolves its own transport errors. Failing open here would be silent, + // so it is resolved as a fault under the configured policy instead. + // + // The policy is re-read rather than defaulted: if `ensureConfig` is + // what failed then `config` is still null, and hard-coding fail-open + // there would override an explicit `NEMO_RELAY_PI_FAIL=closed` -- the + // one case where an operator has asked for exactly this to block. + const detail = error instanceof Error ? error.message : String(error); + const fault = resolveFault( + config ?? configFromEnv(safeSessionId(ctx)), + `the inline-shell gate failed: ${detail}`, + USER_BASH_TOOL_NAME, + ); + if (fault.kind === 'block') return { result: refusalResult(fault.reason) }; + return undefined; + } + }, + ); } /** pi's session id, with a fallback so a missing manager cannot break loading. */ diff --git a/integrations/pi/src/pi-hook-types.ts b/integrations/pi/src/pi-hook-types.ts index c47019e8d..28831e3a9 100644 --- a/integrations/pi/src/pi-hook-types.ts +++ b/integrations/pi/src/pi-hook-types.ts @@ -131,6 +131,44 @@ export type ToolCallEventResult = { reason?: string; }; +/** + * Fired when the user runs a shell command inline with the `!` or `!!` prefix. + * + * This path never reaches the tool registry, so `tool_call` does not see it and + * none of the tool gating applies. `excludeFromContext` is the `!!` form, which + * keeps the command and its output out of the model's context -- including the + * output of a refusal. + * + * It fires in pi's interactive TUI and in RPC mode's `bash` command; there is no + * bang prefix in headless `-p` mode, which has no input loop to type it into. + */ +export type UserBashEvent = { + type: 'user_bash'; + command: string; + excludeFromContext: boolean; + cwd: string; +}; + +/** + * Returning `result` makes pi skip execution and record that result as if the + * command had run; returning `operations` replaces the execution backend but + * keeps the original command. There is no block-and-reason form, so a refusal + * is a synthetic failed result -- see `src/user-bash.ts`. + * + * ⚠️ Unlike `tool_call`, `emitUserBash` wraps handlers in try/catch, so an + * exception here fails **open**: pi logs it and runs the command. And the first + * handler to return anything at all wins, so an earlier-loading extension can + * preempt this one -- `pi -e` loads first, `pi install` loads last. + */ +export type UserBashEventResult = { + result?: { + output: string; + exitCode: number | undefined; + cancelled: boolean; + truncated: boolean; + }; +}; + /** A model, narrowed to the fields this extension reads. */ export type PiModel = { id: string; @@ -176,6 +214,7 @@ export type ExtensionAPI = { on(event: 'tool_execution_start', handler: ExtensionHandler): void; on(event: 'tool_execution_end', handler: ExtensionHandler): void; on(event: 'tool_call', handler: ExtensionHandler): void; + on(event: 'user_bash', handler: ExtensionHandler): void; on(event: 'model_select', handler: ExtensionHandler): void; /** diff --git a/integrations/pi/src/user-bash.ts b/integrations/pi/src/user-bash.ts new file mode 100644 index 000000000..90e63c9d8 --- /dev/null +++ b/integrations/pi/src/user-bash.ts @@ -0,0 +1,120 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * The refusal half of the inline-shell gate. + * + * pi's bang-prefixed shell (`!cmd`, and `!!cmd` to keep the output out of the + * model's context) never touches the tool path: it goes to `emitUserBash`, so + * none of the `tool_call` gating covers it. `user_bash` *is* interceptable -- + * a handler that returns a `BashResult` makes pi skip execution entirely and + * record that result instead -- but pi gives the hook no block-and-reason + * contract the way `tool_call` does. There is no `{block, reason}` here. + * + * So a refusal has to be a **synthetic failed `BashResult`**, and its shape is + * a design decision rather than something pi dictates: + * + * - `exitCode` is {@link REFUSED_EXIT_CODE}, the shell convention for "found, + * but could not be executed". It is not 0 (which would read as success), not + * 1 (which every failing command already uses, so a policy refusal would be + * indistinguishable from the command failing), and not 127 (which means "not + * found" and would send a user hunting for a missing binary). + * - `output` is what the user sees in the terminal, and -- for `!cmd`, though + * not for `!!cmd` -- what lands in the model's context, because pi records a + * returned result through `recordBashResult` exactly as if the command had + * run. It is one attribution line followed by the gateway's reason verbatim, + * on the standing rule that the reason string is a prompt: it reaches a model + * unframed, so it should read as guidance rather than as an error code. + * - `cancelled` and `truncated` are false: nothing was started, so nothing was + * interrupted, and the message is whole. + * + * ⚠️ **Never throw from the handler.** `emitUserBash` wraps handlers in + * try/catch and moves on, so a thrown error fails *open* and the command runs + * unchecked -- the opposite of `tool_call`, which has no try/catch and fails + * closed. Every path here returns an explicit decision. + */ + +/** + * pi's `BashResult`, narrowed to the fields a synthetic refusal sets. + * + * Mirrored from pi `v0.84.0`, `core/bash-executor.ts`. `fullOutputPath` is + * deliberately absent: it points at a temp file holding output too large to + * inline, and a refusal has no output beyond its reason. + */ +export type BashResult = { + output: string; + exitCode: number | undefined; + cancelled: boolean; + truncated: boolean; +}; + +/** + * Exit code reported for a command the gateway refused. + * + * 126 is the POSIX shell convention for a command that was found but could not + * be executed -- a permission problem rather than a missing binary -- which is + * the closest existing meaning to "a policy declined to run this". + */ +export const REFUSED_EXIT_CODE = 126; + +/** + * The tool name an inline shell command is gated under. + * + * Deliberately **not** `bash`. pi's `bash` tool and the bang prefix are two + * different things with different provenance: one is proposed by the model, the + * other is typed by the user. A guardrail only receives the tool name and the + * arguments, so if both arrived as `bash` a policy could not tell them apart -- + * and "the model may not run shell commands" is a common rule that should not + * also stop the human from typing `!git status`. Keeping the names distinct + * makes the policy author choose; the cost is that a policy which wants to + * cover both has to name both, which the docs state. + */ +export const USER_BASH_TOOL_NAME = 'user_bash'; + +/** Hook event posted when the gate opens, and its synthesized close. */ +export const USER_BASH_HOOK = 'user_bash'; +export const USER_BASH_END_HOOK = 'user_bash_end'; + +/** First line of every refusal, so the user knows what declined the command. */ +const ATTRIBUTION = 'NeMo Relay blocked this command.'; + +/** + * Build the result pi records in place of running the command. + * + * The reason is reproduced verbatim after the attribution line; it is the + * guardrail's own words, and for `!cmd` the model reads it. + */ +export function refusalResult(reason: string): BashResult { + return { + output: `${ATTRIBUTION}\n\n${reason}`, + exitCode: REFUSED_EXIT_CODE, + cancelled: false, + truncated: false, + }; +} + +/** + * The reason used when a request intercept rewrote an inline shell command. + * + * pi's `UserBashEventResult` can replace the *result* or supply custom + * execution `operations`, but it cannot replace the command: both call sites + * pass the original string on to `executeBash`, and the terminal component has + * already been built from it. Taking over execution to run the rewrite instead + * would mean reimplementing pi's shell selection, command prefix and + * process-tree cancellation, which is a behaviour change the sidecar has no + * business making. + * + * So the rewrite cannot be honoured, and the command is refused rather than run + * unmodified -- the same rule the tool path already applies to a transform it + * cannot apply safely, and for the same reason: running the original would + * silently discard a policy decision. + */ +export function transformRefusalReason(): string { + return ( + 'A NeMo Relay policy rewrote the arguments for this command, but pi provides no way to ' + + 'execute a rewritten inline shell command -- the bang prefix runs the text you typed. The ' + + 'command was refused rather than run unmodified, because running it would ignore the policy. ' + + 'Re-run it with the change applied by hand, or ask the policy owner to gate the bash tool ' + + 'instead. This is a configuration problem in the policy, not a judgement about the command.' + ); +} diff --git a/integrations/pi/test/user-bash.test.mjs b/integrations/pi/test/user-bash.test.mjs new file mode 100644 index 000000000..5f460831c --- /dev/null +++ b/integrations/pi/test/user-bash.test.mjs @@ -0,0 +1,295 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Drives the inline-shell gate, which has no counterpart on the tool path. + * + * `tool_call` returns `{block, reason}` and pi renders the refusal for us. + * `user_bash` has no such contract: a refusal has to be a synthetic failed + * `BashResult` that pi records as if the command had run, so the *shape* of + * that result is the wire contract here and is pinned below. + * + * The gateway half is pinned in Rust by + * `pi_user_bash_hook_rejects_when_conditional_guardrail_blocks` and + * `pi_user_bash_is_not_gated_by_a_policy_that_names_the_bash_tool` + * (`crates/cli/tests/coverage/shared/server_tests.rs`). + * + * Run: node --test integrations/pi/test/*.test.mjs + */ +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import { after, before, beforeEach, describe, it } from 'node:test'; + +const extension = (await import('../index.ts')).default; +const { REFUSED_EXIT_CODE, refusalResult } = await import('../src/user-bash.ts'); + +/** A stub gateway whose reply for the next request is set per test. */ +function stubGateway() { + const posts = []; + let reply = { status: 200, payload: {} }; + const server = createServer((req, res) => { + let body = ''; + req.on('data', (c) => { + body += c; + }); + req.on('end', () => { + const parsed = JSON.parse(body || '{}'); + posts.push(parsed); + // Only the gate itself is answered specially; the synthesized close and + // every observability post are plain allows. + const isGate = parsed.hook_event_name === 'user_bash'; + const { status, payload } = isGate ? reply : { status: 200, payload: {} }; + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(JSON.stringify(payload)); + }); + }); + return { + server, + posts, + replyWith(next) { + reply = next; + }, + }; +} + +/** Registers the extension and returns a driver that fires hooks in order. */ +function load() { + const handlers = new Map(); + const pi = { + on(name, handler) { + if (!handlers.has(name)) handlers.set(name, []); + handlers.get(name).push(handler); + }, + registerProvider() {}, + }; + extension(pi); + const ctx = { + cwd: '/work', + mode: 'interactive', + hasUI: true, + sessionManager: { getSessionId: () => 'inline-shell-session' }, + }; + return async (name, event = {}) => { + let result; + for (const handler of handlers.get(name) ?? []) { + result = await handler({ type: name, ...event }, ctx); + } + return result; + }; +} + +/** + * Drain the extension's serial post queue. + * + * The gate awaits its own verdict, but the close is enqueued and not awaited -- + * the same treatment every observability post gets. `session_shutdown` awaits + * the chain, which is how the extension itself guarantees nothing is lost on + * exit, so it doubles as the drain here. + */ +const drain = (fire) => fire('session_shutdown', { reason: 'quit' }); + +const named = (posts, name) => posts.filter((p) => p.hook_event_name === name); + +/** The guardrail rejection shape `CliError::into_response` produces, byte for byte. */ +const rejection = (reason) => ({ + status: 403, + payload: { + error: { message: `guardrail rejected: ${reason}`, type: 'nemo_relay_guardrail_rejected', reason }, + }, +}); + +describe('inline shell gate', () => { + let gateway; + let url; + + before(async () => { + gateway = stubGateway(); + await new Promise((r) => gateway.server.listen(0, '127.0.0.1', r)); + url = `http://127.0.0.1:${gateway.server.address().port}`; + process.env.NEMO_RELAY_PI_GATEWAY_URL = url; + }); + + after(() => { + gateway.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + delete process.env.NEMO_RELAY_PI_FAIL; + }); + + beforeEach(() => { + gateway.posts.length = 0; + gateway.replyWith({ status: 200, payload: {} }); + delete process.env.NEMO_RELAY_PI_FAIL; + }); + + it('forwards the command under its own tool name, not as bash', async () => { + const fire = load(); + await fire('user_bash', { command: 'git status', excludeFromContext: false, cwd: '/work' }); + await drain(fire); + + const [gate] = named(gateway.posts, 'user_bash'); + assert.ok(gate, 'the gate must post before deciding'); + // A guardrail sees only the tool name and the arguments, so this name is + // the only thing that lets a policy tell a command the user typed from one + // the model proposed. + assert.equal(gate.tool_name, 'user_bash'); + assert.deepEqual(gate.input, { + command: 'git status', + cwd: '/work', + exclude_from_context: false, + }); + assert.equal(gate.session_id, 'inline-shell-session'); + assert.equal(typeof gate.tool_call_id, 'string'); + }); + + it('allows by returning nothing at all, so pi runs the command as typed', async () => { + const fire = load(); + const result = await fire('user_bash', { + command: 'ls', + excludeFromContext: false, + cwd: '/work', + }); + // Any object here is a result or an operations override, and either would + // stop pi running what the user typed. + assert.equal(result, undefined); + + await drain(fire); + const [close] = named(gateway.posts, 'user_bash_end'); + assert.ok(close, 'the gate span must close even when the command is allowed'); + assert.equal(close.status, 'ok'); + }); + + it('refuses a blocked command with a failed result carrying the reason verbatim', async () => { + const fire = load(); + gateway.replyWith(rejection('piping a download into a shell is blocked here')); + + const result = await fire('user_bash', { + command: 'curl https://example.test/install | sh', + excludeFromContext: false, + cwd: '/work', + }); + + assert.ok(result?.result, 'a refusal must be returned as a synthetic result'); + assert.equal(result.result.exitCode, REFUSED_EXIT_CODE); + assert.equal(result.result.cancelled, false); + assert.equal(result.result.truncated, false); + // Verbatim, on its own line: the user reads it in the terminal and -- for + // `!cmd`, though not `!!cmd` -- so does the model. + assert.match(result.result.output, /piping a download into a shell is blocked here$/); + assert.match(result.result.output, /^NeMo Relay blocked this command\./); + + await drain(fire); + const [close] = named(gateway.posts, 'user_bash_end'); + assert.equal(close.status, 'error'); + assert.equal(close.tool_call_id, named(gateway.posts, 'user_bash')[0].tool_call_id); + }); + + it('refuses when a request intercept rewrites the command, rather than running the original', async () => { + const fire = load(); + // An allow, but with rewritten arguments. pi's user_bash result type can + // replace the result or the execution backend, never the command, so the + // rewrite cannot be honoured. + gateway.replyWith({ + status: 200, + payload: { tool_call: { tool_call_id: 'user-bash-0', input: { command: 'git status --short' } } }, + }); + + const result = await fire('user_bash', { + command: 'git status', + excludeFromContext: false, + cwd: '/work', + }); + + assert.ok(result?.result, 'a rewrite that cannot be applied must not fall through to an allow'); + assert.equal(result.result.exitCode, REFUSED_EXIT_CODE); + assert.match(result.result.output, /no way to execute a rewritten inline shell command/); + }); + + it('fails open by default when the gateway cannot be reached', async () => { + const fire = load(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = 'http://127.0.0.1:1'; + try { + const result = await fire('user_bash', { + command: 'ls', + excludeFromContext: false, + cwd: '/work', + }); + assert.equal(result, undefined, 'a dead sidecar must not brick the user shell'); + } finally { + process.env.NEMO_RELAY_PI_GATEWAY_URL = url; + } + }); + + it('fails closed on demand, and says it is an infrastructure fault rather than a judgement', async () => { + process.env.NEMO_RELAY_PI_FAIL = 'closed'; + process.env.NEMO_RELAY_PI_GATEWAY_URL = 'http://127.0.0.1:1'; + const fire = load(); + try { + const result = await fire('user_bash', { + command: 'ls', + excludeFromContext: false, + cwd: '/work', + }); + assert.ok(result?.result); + assert.equal(result.result.exitCode, REFUSED_EXIT_CODE); + // Telling the user a policy considered and refused their command, when + // nothing did, gives them a false premise to act on. + assert.match(result.result.output, /infrastructure fault, not a judgement/); + } finally { + process.env.NEMO_RELAY_PI_GATEWAY_URL = url; + } + }); + + it('forwards the !! form so a policy can see the output will bypass the model', async () => { + const fire = load(); + await fire('user_bash', { command: 'cat .env', excludeFromContext: true, cwd: '/work' }); + await drain(fire); + assert.equal(named(gateway.posts, 'user_bash')[0].input.exclude_from_context, true); + }); + + // The highest-value assertion in this file. `emitUserBash` wraps handlers in + // try/catch and moves on, so anything that escapes fails *open* and is + // invisible: pi logs an extension error and runs the command unchecked. This + // is the opposite of `tool_call`, which has no try/catch and fails closed. + it('always resolves to a legal decision, under every adverse condition', async () => { + const conditions = [ + ['gateway error', { status: 500, payload: { error: { message: 'kaboom' } } }, url], + ['403 without the guardrail marker', { status: 403, payload: { error: {} } }, url], + ['malformed rejection body', { status: 403, payload: 'not-an-object' }, url], + ['unparseable success body', { status: 200, payload: undefined }, url], + ['unreachable gateway', { status: 200, payload: {} }, 'http://127.0.0.1:1'], + ]; + for (const [label, reply, target] of conditions) { + gateway.replyWith(reply); + process.env.NEMO_RELAY_PI_GATEWAY_URL = target; + const fire = load(); + // `undefined` as a command is the closest thing to an internal failure + // that can be provoked from outside the extension. + const result = await fire('user_bash', { + command: undefined, + excludeFromContext: false, + cwd: '/work', + }).catch((error) => { + assert.fail(`the gate rejected under "${label}", which fails open silently: ${error}`); + }); + assert.ok( + result === undefined || typeof result?.result?.output === 'string', + `"${label}" produced neither an allow nor a refusal: ${JSON.stringify(result)}`, + ); + } + process.env.NEMO_RELAY_PI_GATEWAY_URL = url; + }); +}); + +describe('the synthetic refusal shape', () => { + it('is a failed command, not a successful one and not a generic error', () => { + const result = refusalResult('because policy'); + // 0 would read as success; 1 is what every failing command already returns, + // so a refusal would be indistinguishable from the command failing; 127 + // means "not found" and would send someone hunting for a missing binary. + assert.equal(result.exitCode, 126); + assert.notEqual(result.exitCode, 0); + assert.equal(result.cancelled, false, 'nothing was started, so nothing was interrupted'); + assert.equal(result.truncated, false, 'the message is whole'); + assert.equal(result.output, 'NeMo Relay blocked this command.\n\nbecause policy'); + }); +}); From 18bbf1941047eb2eea3efd5c22e3e2870f18fbcd Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 18:39:03 -0700 Subject: [PATCH 13/41] feat(pi): add a doctor preflight, and harden the integration Three M5 items, all shaped by the same problem: the ways this integration fails are quiet ones. **A doctor preflight for the load path.** pi adds project-scoped extensions to its candidate set only when the project is trusted, and `-p`, `--mode json` and `--mode rpc` never prompt for trust. The skip is a bare conditional rather than an error path, so pi does not treat it as a failure and never reports it -- and the extension cannot report it either, because it is not running. `nemo-relay doctor pi` now warns when an extension sits on a trust-gated path, and probes the gateway the extension will post to. `AgentInfo` gains a `checks` list, empty for Codex and Claude Code and omitted from their JSON entirely: their setup is written by `nemo-relay install`, so `hook_status` already describes it. pi's is installed by the user, wherever they like, and pi's own trust rules decide whether it loads -- a finding that deserves its own status rather than a sentence in a summary. The gateway probe resolves its URL from the resolved `bind` when the environment variable is unset, because the launcher sets that variable *from* the config: a check that only read the variable would report a working gateway as down for anyone who changed `bind`. It classifies rather than just connecting, so "your gateway is down" and "something else owns that port" are told apart, and it never returns `Fail` -- doctor running before the gateway starts is the normal case, not a broken machine. It is skipped under `--offline`, and for an agent that is neither configured nor asked about, so a machine that does not use pi does not spend the timeout budget dialling a gateway nobody mentioned. **Test harness and coverage.** Both test drivers returned the *last* handler's result; pi returns the *first*. That inverted the trap this extension documents in two places, so the harness itself could not catch a regression in preemption behaviour. There is now one shared driver with pi's semantics, and the preemption case is pinned: an extension ahead of ours decides, and the gateway never sees the call. Filled the gaps that left: the `tool_call` gate had no end-to-end test at all -- every component it composes was pinned and the handler wiring them was not, which is easy to miss precisely because the coverage either side looks complete. Also concurrent tools closing out of submission order, unpaired tool boundaries, compaction-driven re-entry, a slow gateway on both gates, and the bound on what an interrupted session loses. **Two limitations documented rather than papered over.** pi registers no SIGINT handler in any mode, so Ctrl+C in a headless mode kills it with teardown never running; what is lost is bounded to marks queued since the last awaited hook, because both gates and both turn boundaries block on their round trip. And a broader one, found while costing the tool-result policy gap: a tool execution intercept registered by any plugin never runs under the CLI gateway. The registry has exactly one consumer, `tool_call_execute`, which the gateway does not call -- it applies policy through the hook path. Guardrails and request intercepts do run there, because both have standalone runners; there is no response-phase equivalent. Worth stating where a user meets it. Also adds `integrations/pi` to the version bump. It is private and unpublished, so this changes nothing today -- it is there so the version cannot already be stale on the day that changes, since a workspace member absent from that list drifts with no lockfile mismatch and no CI failure to catch it. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- crates/cli/src/agents/pi/doctor.rs | 130 +++++++- crates/cli/src/diagnostics/mod.rs | 209 +++++++++++- crates/cli/src/diagnostics/model.rs | 9 + crates/cli/src/diagnostics/render.rs | 13 +- .../tests/coverage/agents/pi_doctor_tests.rs | 158 +++++++++ .../cli/tests/coverage/shared/doctor_tests.rs | 40 ++- docs/nemo-relay-cli/pi.mdx | 60 ++++ integrations/pi/README.md | 61 +++- integrations/pi/test/harness.mjs | 131 ++++++++ integrations/pi/test/lifecycle.test.mjs | 315 +++++++++++++++--- integrations/pi/test/tool-call.test.mjs | 223 +++++++++++++ integrations/pi/test/user-bash.test.mjs | 96 +----- justfile | 4 + 13 files changed, 1300 insertions(+), 149 deletions(-) create mode 100644 crates/cli/tests/coverage/agents/pi_doctor_tests.rs create mode 100644 integrations/pi/test/harness.mjs create mode 100644 integrations/pi/test/tool-call.test.mjs diff --git a/crates/cli/src/agents/pi/doctor.rs b/crates/cli/src/agents/pi/doctor.rs index 19a0ccf5e..27ab98e72 100644 --- a/crates/cli/src/agents/pi/doctor.rs +++ b/crates/cli/src/agents/pi/doctor.rs @@ -5,12 +5,55 @@ //! //! Codex and Claude Code can be checked by inspecting files NeMo Relay wrote //! (generated hook config, a settings base URL). pi's hooks live inside an -//! extension the user loads, so the only thing checkable from here is whether -//! that extension is discoverable. +//! extension the user loads, so what is checkable from here is where that +//! extension sits and whether pi will actually load it. +//! +//! **The failure this module exists for is silent.** pi adds project-scoped +//! extensions to its candidate set only when the project is trusted +//! (`core/package-manager.ts:2394`), and `-p`, `--mode json` and `--mode rpc` +//! never prompt for trust (`docs/security.md:29`). Under the default policy a +//! project-scoped extension is therefore dropped by a bare conditional -- not +//! by an error path, so it never reaches pi's extension-load error list and pi +//! does not consider it a failure. Nothing reports it, and **the extension +//! cannot report it either**: by construction it is not running. A preflight +//! that reads the filesystem is the only place this can be caught. + +use std::path::{Path, PathBuf}; + +use super::launch::{PI_EXTENSION_PATH_ENV, PI_GATEWAY_URL_ENV}; -use std::path::PathBuf; +/// pi's per-user configuration root, `~/.pi/agent` unless overridden. +/// +/// Mirrors `getAgentDir()` (pi `config.ts:515-522`), including the environment +/// override, so the preflight looks where pi will actually look. +const PI_AGENT_DIR_ENV: &str = "PI_CODING_AGENT_DIR"; -use super::launch::PI_EXTENSION_PATH_ENV; +/// pi's configuration directory name, from its `piConfig.configDir`. +const PI_CONFIG_DIR: &str = ".pi"; + +/// Gateway URL the extension falls back to when nothing else resolves one. +/// Kept in step with `configFromEnv` in `integrations/pi/src/gateway-client.ts`. +const DEFAULT_GATEWAY_URL: &str = "http://127.0.0.1:4040"; + +/// How pi reaches an extension, which is what decides whether trust gates it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ExtensionScope { + /// Passed with `-e`, which is what `nemo-relay run --agent pi` does. Loads + /// first in precedence and is never trust-gated, so it works in every mode. + Explicit, + /// Auto-discovered under the user's own config directory. Not trust-gated. + User, + /// Auto-discovered under the project's `.pi/`. **Trust-gated**, and + /// therefore silently skipped in every non-interactive mode. + Project, +} + +/// A place a pi extension was found, and how pi would reach it. +#[derive(Debug, Clone)] +pub(crate) struct ExtensionSite { + pub(crate) path: PathBuf, + pub(crate) scope: ExtensionScope, +} /// Human-readable hook status for `nemo-relay doctor`. pub(crate) fn hook_status() -> Result { @@ -37,3 +80,82 @@ fn extension_location() -> Option { .map(PathBuf::from) .filter(|path| path.exists()) } + +/// The gateway URL the extension will post to. +/// +/// Precedence matters here, and the obvious shortcut is wrong: the extension +/// itself only knows `NEMO_RELAY_PI_GATEWAY_URL` and its own hard-coded default, +/// but the launcher sets that variable *from the resolved configuration*. A +/// preflight that only read the variable would probe `127.0.0.1:4040` for a user +/// who configured a different `bind`, and report their working gateway as down. +/// +/// `bind` is a `SocketAddr`, so it is always a concrete host and port -- a +/// wildcard bind such as `0.0.0.0:4040` is reachable on loopback, which is where +/// pi runs. +pub(crate) fn gateway_url(bind: Option) -> String { + if let Some(url) = std::env::var(PI_GATEWAY_URL_ENV) + .ok() + .map(|url| url.trim_end_matches('/').to_string()) + .filter(|url| !url.is_empty()) + { + return url; + } + match bind { + Some(bind) if bind.ip().is_unspecified() => format!("http://127.0.0.1:{}", bind.port()), + Some(bind) => format!("http://{bind}"), + None => DEFAULT_GATEWAY_URL.to_string(), + } +} + +/// Every place pi could load an extension from, that currently holds one. +/// +/// Deliberately reports *any* auto-discovered entry rather than trying to +/// recognize the NeMo Relay extension by filename: `pi install` renames and +/// nests what it writes, so a filename match would miss the installed layout +/// and quietly report nothing -- which is the failure mode being guarded +/// against. The trust question is a property of the directory, not of the file. +pub(crate) fn extension_sites(cwd: &Path) -> Vec { + let mut sites = Vec::new(); + if let Some(path) = extension_location() { + sites.push(ExtensionSite { + path, + scope: ExtensionScope::Explicit, + }); + } + if let Some(dir) = user_extensions_dir() + && directory_has_entries(&dir) + { + sites.push(ExtensionSite { + path: dir, + scope: ExtensionScope::User, + }); + } + let project_dir = cwd.join(PI_CONFIG_DIR).join("extensions"); + if directory_has_entries(&project_dir) { + sites.push(ExtensionSite { + path: project_dir, + scope: ExtensionScope::Project, + }); + } + sites +} + +/// `~/.pi/agent/extensions`, honoring pi's own directory override. +fn user_extensions_dir() -> Option { + let agent_dir = match std::env::var_os(PI_AGENT_DIR_ENV) { + Some(dir) if !dir.is_empty() => PathBuf::from(dir), + _ => crate::agents::shared::host::home_dir() + .ok()? + .join(PI_CONFIG_DIR) + .join("agent"), + }; + Some(agent_dir.join("extensions")) +} + +fn directory_has_entries(path: &Path) -> bool { + std::fs::read_dir(path).is_ok_and(|mut entries| entries.next().is_some()) +} + +#[cfg(test)] +#[path = "../../../tests/coverage/agents/pi_doctor_tests.rs"] +mod tests; diff --git a/crates/cli/src/diagnostics/mod.rs b/crates/cli/src/diagnostics/mod.rs index 943091456..293c72ec2 100644 --- a/crates/cli/src/diagnostics/mod.rs +++ b/crates/cli/src/diagnostics/mod.rs @@ -146,7 +146,7 @@ pub(crate) async fn collect_report( configured_agents, &plugin_diagnostics, ), - agents: collect_agents(target_agent, &resolved).await, + agents: collect_agents(target_agent, probe_mode, &resolved).await, host_plugins: crate::agents::collect_default_integration_readiness(), observability: collect_observability(&resolved.gateway, probe_mode).await, completions: collect_completions(home.as_deref()), @@ -417,6 +417,7 @@ fn plugin_layer_status( async fn collect_agents( target_agent: Option, + probe_mode: DoctorProbeMode, resolved: &ResolvedConfig, ) -> Vec { let mut out = Vec::with_capacity(CodingAgent::ALL.len()); @@ -424,7 +425,7 @@ async fn collect_agents( if target_agent.is_some_and(|target| target != agent) { continue; } - out.push(collect_agent(agent, target_agent == Some(agent), resolved).await); + out.push(collect_agent(agent, target_agent == Some(agent), probe_mode, resolved).await); } out } @@ -432,6 +433,7 @@ async fn collect_agents( async fn collect_agent( agent: CodingAgent, target_requested: bool, + probe_mode: DoctorProbeMode, resolved: &ResolvedConfig, ) -> AgentInfo { let configured = agent_configured(agent, &resolved.agents); @@ -462,6 +464,11 @@ async fn collect_agent( &mut status, &mut details, ); + let checks = + agent_preflight_checks(agent, probe_mode, configured || target_requested, resolved).await; + for check in &checks { + status = combine_status(status, check.status, configured || target_requested); + } AgentInfo { name: agent.as_arg(), status, @@ -470,7 +477,205 @@ async fn collect_agent( path, version, annotation: details.join("; "), + checks, + } +} + +/// Preflight checks that depend on how the *agent* is set up rather than on +/// anything NeMo Relay wrote. +/// +/// Only pi has any. Codex reads a generated hook config and Claude Code reads a +/// settings file, both written by `nemo-relay install`, so their setup is +/// already fully described by `hook_status`. pi's hooks live in an extension the +/// user installs wherever they like, and pi's own trust rules decide whether it +/// loads -- neither of which is visible from anything NeMo Relay owns. +async fn agent_preflight_checks( + agent: CodingAgent, + probe_mode: DoctorProbeMode, + relevant: bool, + resolved: &ResolvedConfig, +) -> Vec { + if agent != CodingAgent::Pi { + return Vec::new(); + } + let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); + let mut checks = vec![pi_extension_trust_check(&cwd)]; + // The filesystem check is free and always worth running -- a stray project-scoped + // extension is worth knowing about whether or not pi is set up yet. The network probe + // is not: a bare `nemo-relay doctor` on a machine that does not use pi would spend the + // timeout budget dialling a gateway nobody asked about, and warn about it. + if relevant { + checks.push(pi_gateway_reachability_check(probe_mode, resolved).await); + } + checks +} + +/// Warn when an extension sits somewhere pi will silently ignore. +/// +/// The danger is specific: pi adds project-scoped extensions to its candidate +/// set only for a trusted project, and its non-interactive modes never prompt +/// for trust. The skip is a bare conditional rather than an error path, so pi +/// does not treat it as a failure and nothing surfaces it -- and the extension +/// cannot surface it either, because it is not running. This check is the only +/// place a user finds out before wondering why NeMo Relay "does nothing". +fn pi_extension_trust_check(cwd: &Path) -> Check { + const NAME: &str = "pi extension load path"; + let sites = crate::agents::pi::doctor::extension_sites(cwd); + let project_sites: Vec<&crate::agents::pi::doctor::ExtensionSite> = sites + .iter() + .filter(|site| site.scope == crate::agents::pi::doctor::ExtensionScope::Project) + .collect(); + + if let Some(project) = project_sites.first() { + return Check { + name: NAME, + status: Status::Warn, + details: format!( + "{} is project-scoped, so pi loads it only when the project is trusted, and \ + `-p`, `--mode json` and `--mode rpc` never prompt -- it is silently skipped \ + there, with nothing reporting it. Install at user scope \ + (`~/.pi/agent/extensions/`, or `pi install` without `--local`), or launch with \ + `nemo-relay run --agent pi`, which passes `-e` and is never trust-gated", + project.path.display() + ), + }; } + + match sites.first() { + Some(site) => Check { + name: NAME, + status: Status::Pass, + details: format!( + "{} ({})", + site.path.display(), + match site.scope { + crate::agents::pi::doctor::ExtensionScope::Explicit => + "passed with `-e`, which loads first and is never trust-gated", + crate::agents::pi::doctor::ExtensionScope::User => + "user scope, which is never trust-gated", + crate::agents::pi::doctor::ExtensionScope::Project => "project scope", + } + ), + }, + None => Check { + name: NAME, + status: Status::Info, + details: format!( + "no pi extension found; set {} or install one into `~/.pi/agent/extensions/`", + crate::agents::pi::launch::PI_EXTENSION_PATH_ENV + ), + }, + } +} + +/// Probe the gateway the pi extension will post to. +/// +/// Deliberately a warning rather than a failure when it does not answer: the +/// gateway is a separate process a user normally starts alongside pi, so doctor +/// running before it is the common case rather than a broken one. What makes +/// this worth probing at all is that the extension defaults to **failing open**, +/// so an unreachable gateway does not stop pi -- it silently stops enforcing. +async fn pi_gateway_reachability_check( + probe_mode: DoctorProbeMode, + resolved: &ResolvedConfig, +) -> Check { + const NAME: &str = "pi gateway reachability"; + let url = crate::agents::pi::doctor::gateway_url(Some(resolved.gateway.bind)); + if probe_mode.is_offline() { + return Check { + name: NAME, + status: Status::Info, + details: format!("{url} (live reachability probe skipped (--offline))"), + }; + } + // The shared probe classifies rather than just connecting, which is the difference + // between "your gateway is down" and "something else owns that port" -- two problems + // with nothing in common. It is blocking and loopback-only, hence the offload; a pi + // gateway pointed somewhere else falls back to a plain reachability check below. + let probed = if is_loopback(&url) { + let probe_url = url.clone(); + timeout( + NETWORK_TIMEOUT, + tokio::task::spawn_blocking(move || crate::gateway::client::probe(&probe_url, None)), + ) + .await + .ok() + .and_then(Result::ok) + } else { + None + }; + + let unreachable = |detail: &str| Check { + name: NAME, + status: Status::Warn, + details: format!( + "{url} {detail}; start the gateway before pi, or every hook will fault -- and \ + because the extension fails open by default, pi keeps running with no policy applied" + ), + }; + + match probed { + Some(crate::gateway::client::RelayHealth::Compatible) => Check { + name: NAME, + status: Status::Pass, + details: format!("{url} is running a compatible NeMo Relay gateway"), + }, + Some(crate::gateway::client::RelayHealth::Incompatible) => Check { + name: NAME, + status: Status::Warn, + details: format!( + "{url} is a NeMo Relay gateway of an incompatible version; the pi extension \ + posts hooks here" + ), + }, + Some(crate::gateway::client::RelayHealth::Foreign) => Check { + name: NAME, + status: Status::Warn, + details: format!( + "{url} is answering, but it is not a NeMo Relay gateway -- something else owns \ + that port, so every hook goes somewhere unexpected" + ), + }, + Some(crate::gateway::client::RelayHealth::Unavailable) => unreachable("is not answering"), + // Non-loopback, or the blocking probe did not finish in budget. + None => match reqwest::Client::builder() + .timeout(NETWORK_TIMEOUT) + .build() + .ok() + { + Some(client) => match client.get(format!("{url}/healthz")).send().await { + Ok(response) if response.status().is_success() => Check { + name: NAME, + status: Status::Pass, + details: format!("{url} answered /healthz"), + }, + Ok(response) => Check { + name: NAME, + status: Status::Warn, + details: format!( + "{url} answered /healthz with HTTP {}; the pi extension posts hooks here", + response.status().as_u16() + ), + }, + Err(_) => unreachable("did not answer /healthz"), + }, + None => unreachable("could not be probed"), + }, + } +} + +/// Whether a URL names the local machine, which is all the shared probe supports. +fn is_loopback(url: &str) -> bool { + reqwest::Url::parse(url) + .ok() + .and_then(|parsed| parsed.host_str().map(str::to_owned)) + .is_some_and(|host| { + host == "localhost" + || host + .trim_matches(['[', ']']) + .parse::() + .is_ok_and(|ip| ip.is_loopback()) + }) } fn agent_details( diff --git a/crates/cli/src/diagnostics/model.rs b/crates/cli/src/diagnostics/model.rs index 6ca5ca3b2..397f15a00 100644 --- a/crates/cli/src/diagnostics/model.rs +++ b/crates/cli/src/diagnostics/model.rs @@ -154,4 +154,13 @@ pub(crate) struct AgentInfo { pub path: Option, pub version: Option, pub annotation: String, + /// Agent-specific preflight checks, folded into `status` above. + /// + /// Empty for every agent whose setup NeMo Relay writes itself, because + /// `annotation` already says everything there is to say about it. pi is the + /// exception: its hooks live in an extension the *user* installs, so where + /// that extension sits decides whether pi loads it at all, and that is a + /// finding with its own status rather than a sentence in a summary. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub checks: Vec, } diff --git a/crates/cli/src/diagnostics/render.rs b/crates/cli/src/diagnostics/render.rs index 3844d7360..b33f1f6a1 100644 --- a/crates/cli/src/diagnostics/render.rs +++ b/crates/cli/src/diagnostics/render.rs @@ -206,6 +206,14 @@ pub(super) fn format_human_agents(out: &mut String, report: &DoctorReport) { )); } } + for check in &agent.checks { + out.push_str(&format!( + " {} {}: {}\n", + format_status(check.status), + check.name, + check.details + )); + } } out.push('\n'); } @@ -295,7 +303,10 @@ pub(crate) fn format_json(report: &DoctorReport) -> Result { /// the same JSON schema as `doctor.agents` for consistency. pub(crate) async fn agents_report() -> Result, CliError> { let resolved = resolve_server_config(&GatewayOverrides::default())?; - Ok(collect_agents(None, &resolved).await) + // Offline: `nemo-relay agents` is a fast local listing, not a diagnostic run. `doctor` + // is where a user asks for live probes, and it is the command that has `--offline` to + // turn them back off. + Ok(collect_agents(None, DoctorProbeMode::Offline, &resolved).await) } /// Renders the agents listing in human form. diff --git a/crates/cli/tests/coverage/agents/pi_doctor_tests.rs b/crates/cli/tests/coverage/agents/pi_doctor_tests.rs new file mode 100644 index 000000000..14de4cd5c --- /dev/null +++ b/crates/cli/tests/coverage/agents/pi_doctor_tests.rs @@ -0,0 +1,158 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::ffi::OsStr; + +use super::*; +use crate::test_support::EnvScope; + +/// Isolate the three environment variables that steer extension discovery, so a +/// developer's own pi install cannot make these pass or fail. +fn scoped(extension: Option<&OsStr>, agent_dir: Option<&OsStr>) -> EnvScope { + EnvScope::set(&[ + (PI_EXTENSION_PATH_ENV, extension), + (PI_AGENT_DIR_ENV, agent_dir), + ]) +} + +#[test] +fn a_project_scoped_extension_is_reported_because_pi_will_not_say_so() { + let temp = tempfile::tempdir().unwrap(); + let project_extensions = temp.path().join(".pi").join("extensions"); + std::fs::create_dir_all(&project_extensions).unwrap(); + std::fs::write(project_extensions.join("nemo-relay.ts"), "export default 1").unwrap(); + let empty_home = temp.path().join("home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let _env = scoped(None, Some(empty_home.as_os_str())); + let sites = extension_sites(temp.path()); + + // This is the whole point of the check: pi drops this extension with a bare + // conditional in every non-interactive mode, never reports it, and the + // extension cannot report it either because it is not running. + assert!( + sites + .iter() + .any(|site| site.scope == ExtensionScope::Project), + "a project-scoped extension must be reported: {sites:?}" + ); +} + +#[test] +fn an_empty_project_directory_is_not_reported() { + let temp = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(temp.path().join(".pi").join("extensions")).unwrap(); + let empty_home = temp.path().join("home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let _env = scoped(None, Some(empty_home.as_os_str())); + + // A `.pi/extensions` directory that pi created and nothing was ever put in + // is not a finding; warning about it would train users to ignore the check. + assert!( + extension_sites(temp.path()).is_empty(), + "an empty project extensions directory must not be reported" + ); +} + +#[test] +fn the_explicit_path_is_reported_as_ungated() { + let temp = tempfile::tempdir().unwrap(); + let entry = temp.path().join("index.ts"); + std::fs::write(&entry, "export default 1").unwrap(); + let empty_home = temp.path().join("home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let _env = scoped(Some(entry.as_os_str()), Some(empty_home.as_os_str())); + let sites = extension_sites(temp.path()); + + // `-e` loads first in pi's precedence order and survives `--no-extensions`, + // so an extension reached this way is never subject to project trust -- + // which is exactly why the launcher uses it. + assert_eq!(sites.len(), 1, "{sites:?}"); + assert_eq!(sites[0].scope, ExtensionScope::Explicit); + assert_eq!(sites[0].path, entry); +} + +#[test] +fn a_user_scope_install_is_reported_as_ungated() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + let user_extensions = agent_dir.join("extensions"); + std::fs::create_dir_all(&user_extensions).unwrap(); + std::fs::write(user_extensions.join("nemo-relay.ts"), "export default 1").unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = extension_sites(temp.path()); + + assert_eq!(sites.len(), 1, "{sites:?}"); + assert_eq!(sites[0].scope, ExtensionScope::User); +} + +#[test] +fn an_explicit_path_that_does_not_exist_is_not_reported() { + let temp = tempfile::tempdir().unwrap(); + let missing = temp.path().join("gone.ts"); + let empty_home = temp.path().join("home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let _env = scoped(Some(missing.as_os_str()), Some(empty_home.as_os_str())); + + // A stale environment variable is worse than none: it would report an + // ungated load path for a file pi cannot read. + assert!(extension_sites(temp.path()).is_empty()); + assert!(!extension_configured()); +} + +#[test] +fn the_gateway_url_matches_what_the_extension_resolves() { + let _env = EnvScope::set(&[(PI_GATEWAY_URL_ENV, None)]); + // Kept in step with `configFromEnv` in integrations/pi/src/gateway-client.ts; + // a drift here would probe an endpoint the extension never posts to. + assert_eq!(gateway_url(None), "http://127.0.0.1:4040"); +} + +#[test] +fn the_gateway_url_honors_the_launcher_variable_and_strips_trailing_slashes() { + let _env = EnvScope::set(&[( + PI_GATEWAY_URL_ENV, + Some(OsStr::new("http://gateway.test:9999///")), + )]); + assert_eq!(gateway_url(None), "http://gateway.test:9999"); +} + +#[test] +fn the_gateway_url_follows_a_configured_bind_rather_than_the_default_port() { + let _env = EnvScope::set(&[(PI_GATEWAY_URL_ENV, None)]); + // The launcher sets the environment variable *from* the resolved config, so a + // preflight that only read the variable would report a working gateway as down + // for anyone who changed `bind`. + assert_eq!( + gateway_url(Some("127.0.0.1:8123".parse().unwrap())), + "http://127.0.0.1:8123" + ); +} + +#[test] +fn a_wildcard_bind_is_probed_on_loopback_where_pi_actually_runs() { + let _env = EnvScope::set(&[(PI_GATEWAY_URL_ENV, None)]); + // `http://0.0.0.0:4040` is not a dialable address; the gateway bound that way is + // reachable on loopback, which is where the pi extension posts from. + assert_eq!( + gateway_url(Some("0.0.0.0:4040".parse().unwrap())), + "http://127.0.0.1:4040" + ); +} + +#[test] +fn the_environment_variable_wins_over_a_configured_bind() { + let _env = EnvScope::set(&[( + PI_GATEWAY_URL_ENV, + Some(OsStr::new("http://elsewhere:1234")), + )]); + // Someone who set the variable by hand is pointing pi somewhere deliberately. + assert_eq!( + gateway_url(Some("127.0.0.1:4040".parse().unwrap())), + "http://elsewhere:1234" + ); +} diff --git a/crates/cli/tests/coverage/shared/doctor_tests.rs b/crates/cli/tests/coverage/shared/doctor_tests.rs index 601ea8b27..6f98320bf 100644 --- a/crates/cli/tests/coverage/shared/doctor_tests.rs +++ b/crates/cli/tests/coverage/shared/doctor_tests.rs @@ -194,6 +194,7 @@ fn exit_code_fails_when_agent_readiness_fails() { path: None, version: None, annotation: "configured command not found on $PATH".into(), + checks: Vec::new(), }); assert_eq!(exit_code(&report), 1); } @@ -335,6 +336,7 @@ fn format_human_uses_symbols_for_agent_statuses() { path: Some(PathBuf::from("/bin/claude")), version: Some("1.0.0".into()), annotation: "hooks: injected during run".into(), + checks: Vec::new(), }, AgentInfo { name: "codex", @@ -344,6 +346,7 @@ fn format_human_uses_symbols_for_agent_statuses() { path: None, version: None, annotation: "not configured".into(), + checks: Vec::new(), }, ]; @@ -778,7 +781,12 @@ async fn collect_agents_filters_target_and_records_version() { let mut resolved = ResolvedConfig::default(); resolved.agents.codex.command = Some(codex.to_string_lossy().into_owned()); - let agents = collect_agents(Some(CodingAgent::Codex), &resolved).await; + let agents = collect_agents( + Some(CodingAgent::Codex), + DoctorProbeMode::Offline, + &resolved, + ) + .await; assert_eq!(agents.len(), 1); assert_eq!(agents[0].name, "codex"); @@ -803,7 +811,12 @@ async fn collect_agents_preserves_wrapper_argv_for_version_validation() { let mut resolved = ResolvedConfig::default(); resolved.agents.codex.command = Some(format!("{} codex", wrapper.display())); - let agents = collect_agents(Some(CodingAgent::Codex), &resolved).await; + let agents = collect_agents( + Some(CodingAgent::Codex), + DoctorProbeMode::Offline, + &resolved, + ) + .await; assert_eq!(agents[0].status, Status::Pass); assert_eq!(agents[0].path.as_deref(), Some(wrapper.as_path())); @@ -822,12 +835,18 @@ async fn collect_agents_distinguishes_required_and_optional_version_failures() { let mut configured = ResolvedConfig::default(); configured.agents.codex.command = Some(codex.display().to_string()); - let required = collect_agents(Some(CodingAgent::Codex), &configured).await; + let required = collect_agents( + Some(CodingAgent::Codex), + DoctorProbeMode::Offline, + &configured, + ) + .await; assert_eq!(required[0].status, Status::Fail); assert!(required[0].annotation.contains("is unsupported")); let _environment = EnvScope::set(&[("PATH", Some(temp.path().as_os_str()))]); - let discovered = collect_agents(None, &ResolvedConfig::default()).await; + let discovered = + collect_agents(None, DoctorProbeMode::Offline, &ResolvedConfig::default()).await; let optional = discovered .iter() .find(|agent| agent.name == "codex") @@ -837,7 +856,12 @@ async fn collect_agents_distinguishes_required_and_optional_version_failures() { std::fs::write(&codex, "#!/bin/sh\nexit 0\n").unwrap(); make_executable(&codex); - let required = collect_agents(Some(CodingAgent::Codex), &configured).await; + let required = collect_agents( + Some(CodingAgent::Codex), + DoctorProbeMode::Offline, + &configured, + ) + .await; assert_eq!(required[0].status, Status::Fail); assert!( required[0] @@ -845,7 +869,8 @@ async fn collect_agents_distinguishes_required_and_optional_version_failures() { .contains("could not determine version") ); - let discovered = collect_agents(None, &ResolvedConfig::default()).await; + let discovered = + collect_agents(None, DoctorProbeMode::Offline, &ResolvedConfig::default()).await; let optional = discovered .iter() .find(|agent| agent.name == "codex") @@ -2353,6 +2378,7 @@ fn format_agents_human_lists_supported_and_separates_detected() { path: Some(PathBuf::from("/opt/homebrew/bin/claude")), version: Some("2.1.4".into()), annotation: "hooks: injected during run".into(), + checks: Vec::new(), }, AgentInfo { name: "codex", @@ -2362,6 +2388,7 @@ fn format_agents_human_lists_supported_and_separates_detected() { path: None, version: None, annotation: "not configured".into(), + checks: Vec::new(), }, ]; let rendered = format_agents_human(&agents); @@ -2386,6 +2413,7 @@ fn format_agents_json_matches_doctor_agents_shape() { path: Some(PathBuf::from("/opt/homebrew/bin/claude")), version: Some("2.1.4".into()), annotation: "hooks: injected during run".into(), + checks: Vec::new(), }]; let json = format_agents_json(&agents).unwrap(); let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index 87b88799d..c982b4b21 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -55,6 +55,28 @@ Install pi 0.84.0 or newer, and confirm the NeMo Relay extension is reachable: nemo-relay doctor pi ``` +That command reports two things nothing else can: which path the extension will +load from and whether pi will trust it, and whether the gateway is answering at +the URL the extension posts to. Run it first whenever Relay does not seem to be +doing anything. + +### Install At User Scope + +Copy the extension into `~/.pi/agent/extensions/`, or install it with +`pi install ` without `--local`. `nemo-relay run --agent pi` does not +need either — it passes the extension with `-e`, which loads first and is never +trust-gated. + + +**Do not install into `.pi/extensions/` or with `pi install --local`.** pi adds +project-scoped extensions to its candidate set only when the project is trusted, +and `-p`, `--mode json` and `--mode rpc` never prompt for trust. Under the +default policy the extension is dropped by a bare conditional — not an error +path — so pi does not treat it as a failure and never reports it, and the +extension cannot report it either, because it is not running. The only symptom +is that Relay appears to do nothing. `nemo-relay doctor pi` warns about it. + + ## Transparent Run Use the wrapper for no-install local observability. pi has no `nemo-relay pi` @@ -233,6 +255,44 @@ before the span opens, but pi fires `tool_execution_end` for blocked calls too, and the gateway synthesizes the missing pair, tagged `tool_correlation_status: "agent_fallback"`. +## Tool-Result Policy Is Not Available + +Relay's only tool middleware that can change what a tool *returned* is the +execution intercept, which wraps the callback and so owns execution. pi runs its +tools in its own process and reports the outcome, and the gateway builds spans +from hook posts rather than executing anything, so neither side ever holds that +callback. + + +This is broader than pi. **A tool execution intercept registered by any plugin +does not run under the CLI gateway at all** — the registry's only consumer is +`tool_call_execute`, which the gateway never calls, because it applies policy +through the hook path instead. Conditional-execution guardrails and request +intercepts do run there, since both have standalone runners the gateway invokes +directly; there is no response-phase equivalent to invoke. + + +## Interrupted Sessions + +pi registers **no SIGINT handler in any mode**. All three modes install handlers +for `SIGTERM`, plus `SIGHUP` off Windows, and raw mode is set only by the +interactive TUI — so under `-p`, `--mode json` and `--mode rpc`, Ctrl+C is a real +SIGINT that terminates the process with teardown never running. pi's +`session_shutdown` never fires, and the extension never drains its queue. + +The loss is bounded. Every hook the extension *awaits* has already reached the +gateway: both gates and both turn boundaries block on their round trip. An +interrupt can therefore drop only observability marks queued since the last +awaited hook, and the gateway keeps everything already delivered — the session +scope is left open rather than the trace being lost. + +| Exit | Teardown runs? | +| --- | --- | +| `/quit`, normal completion | Yes | +| `SIGTERM`, `SIGHUP` | Yes | +| `SIGINT` (Ctrl+C in a headless mode) | **No** | +| `SIGKILL`, uncaught exception | **No** | + ## Model Redirection pi has no base-URL flag and no generic environment override — it resolves a base diff --git a/integrations/pi/README.md b/integrations/pi/README.md index 908addbc7..96c501134 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -38,9 +38,37 @@ NEMO_RELAY_PI_GATEWAY_URL=http://127.0.0.1:4040 \ ``` `pi -e` is trust-ungated, loads before discovery, and survives -`--no-extensions`, which makes it the reliable way to load this. For everyday -use, install it with `pi install ` or place it in an auto-discovered -directory (`~/.pi/agent/extensions/`, `.pi/extensions/`). +`--no-extensions`, which makes it the reliable way to load this. It is also what +`nemo-relay run --agent pi` uses. + +### Where to install it + +**User scope only.** Copy the extension into `~/.pi/agent/extensions/`, or +install it with `pi install ` *without* `--local`. + +| Path | Install here? | Trust-gated? | +|---|---|---| +| `~/.pi/agent/extensions/` | Yes | No | +| `pi install` (user scope) | Yes | No | +| `-e ` | Per-invocation; what the launcher uses | No | +| `.pi/extensions/` or `pi install --local` | **No** | **Yes** | + +⚠️ **A project-scoped install is silently skipped, and nothing tells you.** pi +adds project extensions to its candidate set only when the project is trusted, +and `-p`, `--mode json` and `--mode rpc` never prompt for trust — so under the +default policy the extension is dropped by a bare conditional. It is not an +error path, so pi does not treat it as a failure and never reports it, and the +extension cannot report it either: by construction it is not running. The +symptom is simply that NeMo Relay appears to do nothing. + +**`nemo-relay doctor pi` is the check for that.** It reports where the extension +sits, whether that path is trust-gated, and whether the gateway is answering: + +```bash +nemo-relay doctor pi +``` + +Run it first whenever Relay does not seem to be doing anything. ### Environment @@ -299,6 +327,33 @@ worth stating separately: a child pi process running this extension resolves its *own* session id and posts under it, so it does not appear as a subagent of the parent. It appears as an unrelated session. +**Tool-result policy, on either side.** Relay's only middleware that can change +what a tool *returned* is the execution intercept, which wraps the callback and +therefore owns execution. pi never hands the callback over — it runs the tool in +its own process and reports the outcome — and neither does the gateway, which +builds spans from hook posts rather than executing anything. + +⚠️ Worth stating in full, because it is not pi-specific and it surprises people: +**a tool execution intercept registered by any plugin never runs under the CLI +gateway.** The registry has exactly one consumer, `tool_call_execute`, and the +gateway does not call it — it uses `tool_call` / `tool_call_end`. Guardrails and +request intercepts do run, because those have standalone runners the gateway +calls directly. There is no response-phase equivalent. + +**Anything still queued when the process is interrupted.** pi registers **no +SIGINT handler in any mode** — all three build `["SIGTERM"]` plus SIGHUP off +Windows — and raw mode is set only by the TUI, so under `-p`, `--mode json` and +`--mode rpc` a user's Ctrl+C is a real SIGINT that terminates with teardown +never running. `session_shutdown` never fires, so the queue is never drained. + +The loss is bounded, and the bound is what makes this liveable: **every awaited +hook has already reached the gateway.** Both gates (`tool_call`, `user_bash`) and +both turn boundaries block on their round trip, so an interrupt can only drop +observability marks queued since the last awaited one. The gateway keeps +everything already delivered — an interrupted session is left *open*, not lost. +SIGTERM, SIGHUP and `/quit` all run teardown normally; SIGINT, SIGKILL and an +uncaught exception do not. + **LLM spans, when redirection is skipped.** They are present whenever the gateway fronts the endpoint the active model would otherwise have called, and absent otherwise — the `model_redirect` mark in the trace names which it was and diff --git a/integrations/pi/test/harness.mjs b/integrations/pi/test/harness.mjs new file mode 100644 index 000000000..75bee6172 --- /dev/null +++ b/integrations/pi/test/harness.mjs @@ -0,0 +1,131 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Shared test harness: a stub gateway, and a driver that fires pi's hooks. + * + * ⚠️ **The driver returns the *first* non-`undefined` result, not the last.** + * That is pi's rule, in both `emitToolCall` and `emitUserBash`, and it is the + * difference between "our gate decided" and "an extension ahead of us decided + * and we never ran". A last-wins driver silently inverts the trap this + * extension documents in two places, and would let a regression that breaks + * preemption behaviour pass. + */ +import { createServer } from 'node:http'; + +/** + * A gateway whose reply to the *gated* hook is set per test. + * + * Every other post is answered 200 `{}`: they are observability, and answering + * them specially would mean a test asserting on a block could not tell which + * post the block came from. + * + * @param gatedHook the `hook_event_name` whose reply `replyWith` controls + */ +export function stubGateway(gatedHook) { + const posts = []; + let reply = { status: 200, payload: {} }; + const server = createServer((req, res) => { + let body = ''; + req.on('data', (chunk) => { + body += chunk; + }); + req.on('end', () => { + const parsed = JSON.parse(body || '{}'); + posts.push(parsed); + const gated = gatedHook !== undefined && parsed.hook_event_name === gatedHook; + const { status, payload, delayMs } = gated ? reply : { status: 200, payload: {} }; + const send = () => { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(JSON.stringify(payload ?? {})); + }; + if (delayMs) setTimeout(send, delayMs); + else send(); + }); + }); + return { + server, + posts, + replyWith(next) { + reply = next; + }, + reset() { + posts.length = 0; + reply = { status: 200, payload: {} }; + }, + }; +} + +/** Start a stub server on a free port and return its base URL. */ +export async function listen(server) { + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + return `http://127.0.0.1:${server.address().port}`; +} + +/** + * Register the extension and return a driver that fires hooks the way pi does. + * + * `before` registers handlers *ahead* of the extension, which is the + * `pi install` load order: package-sourced extensions load last, so anything + * already installed answers first. Loading with `-e` inverts it, and is what + * `nemo-relay run --agent pi` uses precisely so the gate runs first. + * + * @param extension the extension factory under test + * @param options.before `{ [hookName]: handler }` registered before the extension + * @param options.ctx overrides merged into the extension context + */ +export function load(extension, options = {}) { + const handlers = new Map(); + const register = (name, handler) => { + if (!handlers.has(name)) handlers.set(name, []); + handlers.get(name).push(handler); + }; + for (const [name, handler] of Object.entries(options.before ?? {})) { + register(name, handler); + } + extension({ + on: register, + registerProvider() {}, + }); + const ctx = { + cwd: '/work', + mode: 'interactive', + hasUI: true, + sessionManager: { getSessionId: () => 'sess-under-test' }, + ...options.ctx, + }; + return async (name, event = {}) => { + for (const handler of handlers.get(name) ?? []) { + const result = await handler({ type: name, ...event }, ctx); + // First truthy result wins and stops iteration -- pi's rule. Note that + // `{}` is truthy: an allow must be `undefined`, or it silently preempts + // every extension behind it while deciding nothing. + if (result !== undefined) return result; + } + return undefined; + }; +} + +/** + * Drain the extension's serial post queue. + * + * Gating hooks await their own verdict, but everything else is enqueued and not + * awaited. `session_shutdown` awaits the chain -- that is how the extension + * guarantees nothing is lost on exit -- so it doubles as the drain. + */ +export const drain = (fire) => fire('session_shutdown', { reason: 'quit' }); + +/** Every post with a given hook event name, in arrival order. */ +export const named = (posts, name) => posts.filter((post) => post.hook_event_name === name); + +/** The 403 body `CliError::into_response` produces, byte for byte. */ +export const rejection = (reason) => ({ + status: 403, + payload: { + error: { + message: `guardrail rejected: ${reason}`, + type: 'nemo_relay_guardrail_rejected', + reason, + }, + }, +}); diff --git a/integrations/pi/test/lifecycle.test.mjs b/integrations/pi/test/lifecycle.test.mjs index 223bf5b55..4878ee208 100644 --- a/integrations/pi/test/lifecycle.test.mjs +++ b/integrations/pi/test/lifecycle.test.mjs @@ -13,54 +13,14 @@ * Run: node --test integrations/pi/test/*.test.mjs */ import assert from 'node:assert/strict'; -import { createServer } from 'node:http'; import { after, before, beforeEach, describe, it } from 'node:test'; -const extension = (await import('../index.ts')).default; +import { listen, load as loadExtension, named, stubGateway } from './harness.mjs'; -/** Collects every payload posted to /hooks/pi. */ -function stubGateway() { - const posts = []; - const server = createServer((req, res) => { - let body = ''; - req.on('data', (c) => { - body += c; - }); - req.on('end', () => { - posts.push(JSON.parse(body || '{}')); - res.writeHead(200, { 'content-type': 'application/json' }); - res.end('{}'); - }); - }); - return { server, posts }; -} - -/** Registers the extension and returns a driver that fires hooks in order. */ -function load() { - const handlers = new Map(); - const pi = { - on(name, handler) { - if (!handlers.has(name)) handlers.set(name, []); - handlers.get(name).push(handler); - }, - }; - extension(pi); - const ctx = { - cwd: '/work', - mode: 'print', - hasUI: false, - sessionManager: { getSessionId: () => 'sess-under-test' }, - }; - return async (name, event = {}) => { - let result; - for (const handler of handlers.get(name) ?? []) { - result = await handler({ type: name, ...event }, ctx); - } - return result; - }; -} +const extension = (await import('../index.ts')).default; -const named = (posts, name) => posts.filter((p) => p.hook_event_name === name); +// Headless: pi's print mode has no UI, and none of these hooks depend on one. +const load = () => loadExtension(extension, { ctx: { mode: 'print', hasUI: false } }); describe('lifecycle identity the gateway cannot infer', () => { let ctx; @@ -68,8 +28,7 @@ describe('lifecycle identity the gateway cannot infer', () => { before(async () => { ctx = stubGateway(); - await new Promise((r) => ctx.server.listen(0, '127.0.0.1', r)); - url = `http://127.0.0.1:${ctx.server.address().port}`; + url = await listen(ctx.server); process.env.NEMO_RELAY_PI_GATEWAY_URL = url; }); @@ -180,8 +139,7 @@ describe('session_shutdown reason', () => { before(async () => { ctx = stubGateway(); - await new Promise((r) => ctx.server.listen(0, '127.0.0.1', r)); - process.env.NEMO_RELAY_PI_GATEWAY_URL = `http://127.0.0.1:${ctx.server.address().port}`; + process.env.NEMO_RELAY_PI_GATEWAY_URL = await listen(ctx.server); }); after(() => { @@ -226,8 +184,7 @@ describe('attribution on every hook that has one', () => { before(async () => { ctx = stubGateway(); - await new Promise((r) => ctx.server.listen(0, '127.0.0.1', r)); - process.env.NEMO_RELAY_PI_GATEWAY_URL = `http://127.0.0.1:${ctx.server.address().port}`; + process.env.NEMO_RELAY_PI_GATEWAY_URL = await listen(ctx.server); }); after(() => { @@ -314,8 +271,7 @@ describe('compaction', () => { before(async () => { ctx = stubGateway(); - await new Promise((r) => ctx.server.listen(0, '127.0.0.1', r)); - process.env.NEMO_RELAY_PI_GATEWAY_URL = `http://127.0.0.1:${ctx.server.address().port}`; + process.env.NEMO_RELAY_PI_GATEWAY_URL = await listen(ctx.server); }); after(() => { @@ -385,3 +341,258 @@ describe('compaction', () => { assert.equal(named(ctx.posts, 'session_before_compact')[0].will_retry, true); }); }); + +describe('concurrent tools in one turn', () => { + let ctx; + + before(async () => { + ctx = stubGateway(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = await listen(ctx.server); + }); + + after(() => { + ctx.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + }); + + beforeEach(() => { + ctx.posts.length = 0; + }); + + // pi preflights sibling calls sequentially and then executes them concurrently, so + // `tool_execution_end` arrives in an order that has nothing to do with submission. + // `toolCallId` is the only correlator pi gives, and every piece of per-call state is + // keyed by it -- a regression that keyed on anything else would pass every other test + // here, because no other test uses more than one call id. + it('keeps two calls distinct when their ends arrive out of submission order', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 1 }); + + await fire('tool_execution_start', { toolCallId: 'a', toolName: 'read', args: {} }); + await fire('tool_execution_start', { toolCallId: 'b', toolName: 'bash', args: {} }); + // Both gates in flight at once, which is what the serial queue has to survive. + await Promise.all([ + fire('tool_call', { toolCallId: 'a', toolName: 'read', input: { path: 'a.txt' } }), + fire('tool_call', { toolCallId: 'b', toolName: 'bash', input: { command: 'ls' } }), + ]); + // Closing in the reverse order, which is the case that motivates the id keying. + await fire('tool_execution_end', { toolCallId: 'b', toolName: '', result: 'ok', isError: false }); + await fire('tool_execution_end', { toolCallId: 'a', toolName: '', result: 'ok', isError: false }); + await fire('turn_end', { turnIndex: 0 }); + await fire('session_shutdown', { reason: 'quit' }); + + const ends = named(ctx.posts, 'tool_execution_end'); + assert.equal(ends.length, 2); + // `toolName` was empty on both ends, so each had to be recovered from the start that + // named it -- and recovered from the *right* one. + const byId = Object.fromEntries(ends.map((post) => [post.tool_call_id, post.tool_name])); + assert.deepEqual(byId, { a: 'read', b: 'bash' }); + assert.ok( + ends.every((post) => post.turn_seq === 0), + 'both calls ran in the same turn, so both must carry that turn', + ); + }); + + it('serializes posts even when gating hooks run concurrently', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('turn_start', { turnIndex: 0, timestamp: 1 }); + await Promise.all([ + fire('tool_call', { toolCallId: 'a', toolName: 'read', input: {} }), + fire('tool_call', { toolCallId: 'b', toolName: 'read', input: {} }), + ]); + await fire('session_shutdown', { reason: 'quit' }); + + // The gateway derives boundaries from arrival order, so two concurrent gates must + // still reach it after the turn that owns them. + const order = ctx.posts.map((post) => post.hook_event_name); + assert.ok( + order.indexOf('turn_start') < order.indexOf('tool_call'), + `a tool span must not open before its turn: ${order.join(', ')}`, + ); + assert.equal(named(ctx.posts, 'tool_call').length, 2); + }); +}); + +describe('unpaired tool boundaries', () => { + let ctx; + + before(async () => { + ctx = stubGateway(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = await listen(ctx.server); + }); + + after(() => { + ctx.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + }); + + beforeEach(() => { + ctx.posts.length = 0; + }); + + // Deliberate asymmetry, and one a future contributor would otherwise "fix": + // `tool_execution_start` fires before validation and for calls pi then discards, so + // forwarding it as a tool start would open gateway spans for calls that never ran. + it('never forwards tool_execution_start, which fires for calls that never execute', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('tool_execution_start', { toolCallId: 'ghost', toolName: 'read', args: {} }); + await fire('session_shutdown', { reason: 'quit' }); + + assert.equal(named(ctx.posts, 'tool_execution_start').length, 0); + assert.equal(named(ctx.posts, 'tool_call').length, 0); + }); + + // The reason this hook closes the span rather than `tool_result`: a blocked call takes + // pi's immediate path and never reaches `afterToolCall`, so `tool_result` never fires -- + // but `tool_execution_end` always does, with `isError: true`. + it('closes a blocked call, which never reaches tool_result', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('tool_execution_start', { toolCallId: 'blocked', toolName: 'read', args: {} }); + await fire('tool_execution_end', { + toolCallId: 'blocked', + toolName: '', + result: 'Tool failed.', + isError: true, + }); + await fire('session_shutdown', { reason: 'quit' }); + + const [end] = named(ctx.posts, 'tool_execution_end'); + assert.ok(end, 'a blocked call must still close'); + assert.equal(end.status, 'error'); + assert.equal(end.tool_name, 'read', 'the name comes from the start, which did fire'); + }); + + it('falls back to a placeholder when no start ever named the tool', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('tool_execution_end', { toolCallId: 'orphan', toolName: '', result: null, isError: false }); + await fire('session_shutdown', { reason: 'quit' }); + + // Dropping the post would lose the span entirely; the gateway synthesizes the missing + // start, and a named-but-unknown tool is more useful than nothing. + const [end] = named(ctx.posts, 'tool_execution_end'); + assert.equal(end.tool_name, 'unknown'); + }); +}); + +describe('compaction-driven re-entry', () => { + let ctx; + + before(async () => { + ctx = stubGateway(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = await listen(ctx.server); + }); + + after(() => { + ctx.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + }); + + beforeEach(() => { + ctx.posts.length = 0; + }); + + // The individual fields are pinned elsewhere; this pins the *sequence*, which is the + // one path where `willRetry` is genuine advance notice that the run is about to + // re-enter, and where pi's turn_index restarts at 0 for the second time. + it('keeps counting turns across a compaction, where pi restarts its own index', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 1 }); + await fire('turn_end', { turnIndex: 0 }); + await fire('session_before_compact', { + reason: 'overflow', + willRetry: true, + preparation: { tokensBefore: 120_000, isSplitTurn: false }, + }); + await fire('agent_end', { messages: [] }); + await fire('session_compact', { reason: 'overflow', willRetry: true, fromExtension: false }); + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 2 }); + await fire('turn_end', { turnIndex: 0 }); + await fire('agent_end', { messages: [] }); + await fire('agent_settled'); + await fire('session_shutdown', { reason: 'quit' }); + + const starts = named(ctx.posts, 'turn_start'); + // pi says turn 0 both times; only turn_seq can tell them apart. + assert.deepEqual(starts.map((post) => post.turn_index), [0, 0]); + assert.deepEqual(starts.map((post) => post.turn_seq), [0, 1]); + assert.deepEqual(starts.map((post) => post.attempt_index), [0, 1]); + + const [announced] = named(ctx.posts, 'session_before_compact'); + assert.equal(announced.will_retry, true); + assert.equal(announced.tokens_before, 120_000); + assert.equal(announced.attempt_index, 0, 'announced during the first attempt'); + + const [settled] = named(ctx.posts, 'agent_settled'); + assert.equal(settled.attempts, 2, 'the compaction re-entry is a second attempt'); + }); +}); + +describe('what an interrupted session loses', () => { + let ctx; + + before(async () => { + ctx = stubGateway(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = await listen(ctx.server); + }); + + after(() => { + ctx.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + }); + + beforeEach(() => { + ctx.posts.length = 0; + }); + + // pi registers no SIGINT handler in any mode -- all three build `["SIGTERM"]` plus + // SIGHUP off-Windows -- and raw mode is set only by the TUI, so under `-p`, + // `--mode json` and `--mode rpc` a user's Ctrl+C is a real SIGINT that terminates + // with teardown never running. `session_shutdown` never fires, the drain never + // happens, and whatever is still queued is lost. + // + // This pins the *bound* on that loss: every awaited hook has already reached the + // gateway by the time pi continues, so an interrupt can only drop observability + // marks queued since the last awaited one. Both gates and both turn boundaries + // await; nothing else does. + it('has already delivered every awaited hook, so only queued marks can be lost', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('agent_start'); + + // Awaited: a turn boundary defines what later spans parent to. + await fire('turn_start', { turnIndex: 0, timestamp: 1 }); + assert.equal( + named(ctx.posts, 'turn_start').length, + 1, + 'turn_start must be delivered before pi continues, not queued', + ); + + // Awaited: the gate cannot decide without a verdict. + await fire('tool_call', { toolCallId: 'c1', toolName: 'read', input: { path: 'a.txt' } }); + assert.equal(named(ctx.posts, 'tool_call').length, 1, 'the gate is a round trip, by design'); + + await fire('turn_end', { turnIndex: 0 }); + assert.equal(named(ctx.posts, 'turn_end').length, 1); + + // The interrupt: no session_shutdown, so no drain. What is already delivered stays + // delivered -- the gateway holds it -- so the trace is left open, not lost. + const atInterrupt = ctx.posts.map((post) => post.hook_event_name); + assert.ok(atInterrupt.includes('session_start')); + assert.ok(atInterrupt.includes('turn_start')); + assert.ok(atInterrupt.includes('tool_call')); + assert.ok(atInterrupt.includes('turn_end')); + assert.ok( + !atInterrupt.includes('session_shutdown'), + 'the whole point: an interrupted session never closes', + ); + }); +}); diff --git a/integrations/pi/test/tool-call.test.mjs b/integrations/pi/test/tool-call.test.mjs new file mode 100644 index 000000000..efe463400 --- /dev/null +++ b/integrations/pi/test/tool-call.test.mjs @@ -0,0 +1,223 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Drives the `tool_call` gate end to end. + * + * Every component this handler composes was already pinned -- the 403 shape in + * `gateway-client.test.mjs`, the transform decision matrix in + * `argument-transform.test.mjs`, the gateway's half in Rust -- and the handler + * that wires them together had no test at all. That is an easy gap to miss in + * review precisely because the coverage either side of it looks complete, and + * it is the primary governance seam: for a model-invoked tool, this is the only + * pre-execution decision point that sees arguments. + * + * Run: node --test integrations/pi/test/*.test.mjs + */ +import assert from 'node:assert/strict'; +import { after, before, beforeEach, describe, it } from 'node:test'; + +import { drain, listen, load, named, rejection, stubGateway } from './harness.mjs'; + +const extension = (await import('../index.ts')).default; + +const call = (input = { path: '/work/README.md' }) => ({ + toolCallId: 'c1', + toolName: 'read', + input, +}); + +describe('the tool_call gate', () => { + let gateway; + let url; + + before(async () => { + gateway = stubGateway('tool_call'); + url = await listen(gateway.server); + process.env.NEMO_RELAY_PI_GATEWAY_URL = url; + }); + + after(() => { + gateway.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + delete process.env.NEMO_RELAY_PI_FAIL; + }); + + beforeEach(() => { + gateway.reset(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = url; + delete process.env.NEMO_RELAY_PI_FAIL; + }); + + it('blocks a guardrail rejection and hands pi the reason verbatim', async () => { + const fire = load(extension); + gateway.replyWith(rejection('read .env is blocked; use .env.example')); + + const result = await fire('tool_call', call({ path: '/work/.env' })); + + // pi passes this string to the model with no framing at all, so anything + // added here is read by the model as part of the policy's own words. + assert.deepEqual(result, { + block: true, + reason: 'read .env is blocked; use .env.example', + }); + }); + + it('allows by returning undefined, never a truthy object', async () => { + const fire = load(extension); + const result = await fire('tool_call', call()); + // A truthy result without `block` decides nothing, but it stops every + // later extension's `tool_call` handler from running. + assert.equal(result, undefined); + }); + + it('applies a rewrite to the object pi will execute, and records that it did', async () => { + const fire = load(extension); + gateway.replyWith({ + status: 200, + payload: { tool_call: { tool_call_id: 'c1', input: { path: '/work/.env.example' } } }, + }); + + const event = call({ path: '/work/.env' }); + const result = await fire('tool_call', event); + + assert.equal(result, undefined, 'a rewrite is an allow, not a block'); + // In place, not replaced: pi hands the same object to the tool and to every + // later handler, so a new reference would simply be dropped. + assert.deepEqual(event.input, { path: '/work/.env.example' }); + + await drain(fire); + const [recorded] = named(gateway.posts, 'tool_arguments_transformed'); + assert.ok(recorded, 'the trace must record that the arguments were not the ones proposed'); + assert.equal(recorded.tool_call_id, 'c1'); + assert.equal(recorded.tool_name, 'read'); + }); + + it('blocks a rewrite it cannot apply safely rather than running the original', async () => { + const fire = load(extension); + gateway.replyWith({ + status: 200, + payload: { + tool_call: { tool_call_id: 'c1', input: { path: '/work/.env', sudo: true } }, + }, + }); + + const event = call({ path: '/work/.env' }); + const result = await fire('tool_call', event); + + assert.equal(result?.block, true); + assert.match(result.reason, /added sudo/); + // Falling back to the original arguments would silently discard a policy + // decision, which is the failure the transform exists to prevent. + assert.match(result.reason, /not a judgement about your request/); + assert.deepEqual(event.input, { path: '/work/.env' }, 'a refused rewrite must not be applied'); + }); + + it('does not post a transform mark when nothing rewrote the arguments', async () => { + const fire = load(extension); + await fire('tool_call', call()); + await drain(fire); + assert.equal(named(gateway.posts, 'tool_arguments_transformed').length, 0); + }); + + it('fails open by default when the gateway is unreachable', async () => { + process.env.NEMO_RELAY_PI_GATEWAY_URL = 'http://127.0.0.1:1'; + const fire = load(extension); + // A dead sidecar must not brick the agent, matching how the shipped + // hooks.json files use --fail-open everywhere except pre-tool events. + assert.equal(await fire('tool_call', call()), undefined); + }); + + it('fails closed on demand, and says the block is infrastructure and not policy', async () => { + process.env.NEMO_RELAY_PI_GATEWAY_URL = 'http://127.0.0.1:1'; + process.env.NEMO_RELAY_PI_FAIL = 'closed'; + const fire = load(extension); + + const result = await fire('tool_call', call()); + + assert.equal(result?.block, true); + // Telling the model a policy considered and refused its call, when nothing + // did, gives it a false premise to reason from. + assert.match(result.reason, /infrastructure fault, not a judgement/); + }); + + it('treats a 403 without the guardrail marker as a fault, not a policy decision', async () => { + const fire = load(extension); + gateway.replyWith({ status: 403, payload: { error: { message: 'nope' } } }); + // An authorization failure is not a verdict about the request, so under the + // default fail-open policy the call proceeds rather than being reported to + // the model as refused. + assert.equal(await fire('tool_call', call()), undefined); + }); + + it('resolves a slow gateway through the failure policy rather than hanging pi', async () => { + process.env.NEMO_RELAY_PI_TIMEOUT_MS = '50'; + try { + const fire = load(extension); + gateway.replyWith({ status: 200, payload: {}, delayMs: 400 }); + // pi awaits this handler on its critical path, so a gateway that never + // answers must become a decision, not a stall. + assert.equal(await fire('tool_call', call()), undefined); + } finally { + delete process.env.NEMO_RELAY_PI_TIMEOUT_MS; + } + }); +}); + +describe('an extension ahead of the gate', () => { + let gateway; + let url; + + before(async () => { + gateway = stubGateway('tool_call'); + url = await listen(gateway.server); + process.env.NEMO_RELAY_PI_GATEWAY_URL = url; + }); + + after(() => { + gateway.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + }); + + beforeEach(() => gateway.reset()); + + // The documented blind spot, pinned as behaviour rather than prose: pi stops + // at the first handler that returns anything, so on the `pi install` path -- + // which loads last -- the call is blocked and the gateway never learns it + // happened. `-e` inverts the order, which is why the launcher uses it. + it('preempts the tool gate, and the gateway never sees the call', async () => { + const fire = load(extension, { + before: { tool_call: async () => ({ block: true, reason: 'blocked by another extension' }) }, + }); + + const result = await fire('tool_call', call()); + + assert.deepEqual(result, { block: true, reason: 'blocked by another extension' }); + await drain(fire); + assert.equal( + named(gateway.posts, 'tool_call').length, + 0, + 'the gate never ran, so no policy was consulted and nothing was recorded', + ); + }); + + it('preempts the inline-shell gate the same way', async () => { + const fire = load(extension, { + before: { + user_bash: async () => ({ + result: { output: 'handled elsewhere', exitCode: 0, cancelled: false, truncated: false }, + }), + }, + }); + + const result = await fire('user_bash', { + command: 'git status', + excludeFromContext: false, + cwd: '/work', + }); + + assert.equal(result.result.exitCode, 0); + await drain(fire); + assert.equal(named(gateway.posts, 'user_bash').length, 0); + }); +}); diff --git a/integrations/pi/test/user-bash.test.mjs b/integrations/pi/test/user-bash.test.mjs index 5f460831c..f2537ca28 100644 --- a/integrations/pi/test/user-bash.test.mjs +++ b/integrations/pi/test/user-bash.test.mjs @@ -17,95 +17,22 @@ * Run: node --test integrations/pi/test/*.test.mjs */ import assert from 'node:assert/strict'; -import { createServer } from 'node:http'; import { after, before, beforeEach, describe, it } from 'node:test'; +import { drain, listen, load as loadExtension, named, rejection, stubGateway } from './harness.mjs'; + const extension = (await import('../index.ts')).default; const { REFUSED_EXIT_CODE, refusalResult } = await import('../src/user-bash.ts'); -/** A stub gateway whose reply for the next request is set per test. */ -function stubGateway() { - const posts = []; - let reply = { status: 200, payload: {} }; - const server = createServer((req, res) => { - let body = ''; - req.on('data', (c) => { - body += c; - }); - req.on('end', () => { - const parsed = JSON.parse(body || '{}'); - posts.push(parsed); - // Only the gate itself is answered specially; the synthesized close and - // every observability post are plain allows. - const isGate = parsed.hook_event_name === 'user_bash'; - const { status, payload } = isGate ? reply : { status: 200, payload: {} }; - res.writeHead(status, { 'content-type': 'application/json' }); - res.end(JSON.stringify(payload)); - }); - }); - return { - server, - posts, - replyWith(next) { - reply = next; - }, - }; -} - -/** Registers the extension and returns a driver that fires hooks in order. */ -function load() { - const handlers = new Map(); - const pi = { - on(name, handler) { - if (!handlers.has(name)) handlers.set(name, []); - handlers.get(name).push(handler); - }, - registerProvider() {}, - }; - extension(pi); - const ctx = { - cwd: '/work', - mode: 'interactive', - hasUI: true, - sessionManager: { getSessionId: () => 'inline-shell-session' }, - }; - return async (name, event = {}) => { - let result; - for (const handler of handlers.get(name) ?? []) { - result = await handler({ type: name, ...event }, ctx); - } - return result; - }; -} - -/** - * Drain the extension's serial post queue. - * - * The gate awaits its own verdict, but the close is enqueued and not awaited -- - * the same treatment every observability post gets. `session_shutdown` awaits - * the chain, which is how the extension itself guarantees nothing is lost on - * exit, so it doubles as the drain here. - */ -const drain = (fire) => fire('session_shutdown', { reason: 'quit' }); - -const named = (posts, name) => posts.filter((p) => p.hook_event_name === name); - -/** The guardrail rejection shape `CliError::into_response` produces, byte for byte. */ -const rejection = (reason) => ({ - status: 403, - payload: { - error: { message: `guardrail rejected: ${reason}`, type: 'nemo_relay_guardrail_rejected', reason }, - }, -}); +const load = () => loadExtension(extension); describe('inline shell gate', () => { let gateway; let url; before(async () => { - gateway = stubGateway(); - await new Promise((r) => gateway.server.listen(0, '127.0.0.1', r)); - url = `http://127.0.0.1:${gateway.server.address().port}`; + gateway = stubGateway('user_bash'); + url = await listen(gateway.server); process.env.NEMO_RELAY_PI_GATEWAY_URL = url; }); @@ -116,8 +43,8 @@ describe('inline shell gate', () => { }); beforeEach(() => { - gateway.posts.length = 0; - gateway.replyWith({ status: 200, payload: {} }); + gateway.reset(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = url; delete process.env.NEMO_RELAY_PI_FAIL; }); @@ -137,7 +64,9 @@ describe('inline shell gate', () => { cwd: '/work', exclude_from_context: false, }); - assert.equal(gate.session_id, 'inline-shell-session'); + // The gateway strips inbound routing-identity headers, so the session id in + // the payload is the only correlator that survives. + assert.equal(gate.session_id, 'sess-under-test'); assert.equal(typeof gate.tool_call_id, 'string'); }); @@ -257,7 +186,11 @@ describe('inline shell gate', () => { ['malformed rejection body', { status: 403, payload: 'not-an-object' }, url], ['unparseable success body', { status: 200, payload: undefined }, url], ['unreachable gateway', { status: 200, payload: {} }, 'http://127.0.0.1:1'], + // pi builds its terminal component only after this handler resolves, so a gateway + // that never answers shows the user nothing at all until the timeout fires. + ['slow gateway', { status: 200, payload: {}, delayMs: 400 }, url], ]; + process.env.NEMO_RELAY_PI_TIMEOUT_MS = '50'; for (const [label, reply, target] of conditions) { gateway.replyWith(reply); process.env.NEMO_RELAY_PI_GATEWAY_URL = target; @@ -277,6 +210,7 @@ describe('inline shell gate', () => { ); } process.env.NEMO_RELAY_PI_GATEWAY_URL = url; + delete process.env.NEMO_RELAY_PI_TIMEOUT_MS; }); }); diff --git a/justfile b/justfile index 85272e3d9..855097855 100644 --- a/justfile +++ b/justfile @@ -580,6 +580,10 @@ set_node_package_versions() { set_npm_package_version crates/node/package.json package-lock.json "$version" crates/node set_npm_package_version integrations/openclaw/package.json package-lock.json "$version" integrations/openclaw set_npm_package_dependency_version integrations/openclaw/package.json package-lock.json integrations/openclaw nemo-relay-node "$version" + # `nemo-relay-pi` is private and not published, so this bump changes nothing today. It is here + # so the version cannot already be stale on the day that changes: a workspace member absent + # from this list drifts silently, with no lockfile mismatch and no CI failure to catch it. + set_npm_package_version integrations/pi/package.json package-lock.json "$version" integrations/pi } set_node_package_version() { From e4a8c39bc6bdc02e4faf2282bf14f3d7ed899d12 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 19:23:57 -0700 Subject: [PATCH 14/41] docs(pi): record that the extension is not published to npm `pi install` resolves a local path or a git URL as readily as an `npm:` specifier, so the package being unpublished does not cost a route -- it costs a spelling of one. Publishing would buy that spelling in exchange for an npm namespace, a build step (the sources are TypeScript nothing compiles today) and release wiring, so `private: true` stays, and now says so on purpose rather than reading as an oversight. Both install routes are spelled out with the commands to run, since "user scope" was previously stated as a rule without showing what it looks like. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- docs/nemo-relay-cli/pi.mdx | 20 ++++++++++++++++---- integrations/pi/README.md | 21 ++++++++++++++++++--- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index c982b4b21..77dbe9e49 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -62,10 +62,22 @@ doing anything. ### Install At User Scope -Copy the extension into `~/.pi/agent/extensions/`, or install it with -`pi install ` without `--local`. `nemo-relay run --agent pi` does not -need either — it passes the extension with `-e`, which loads first and is never -trust-gated. +Two routes, both untrusted-project-proof: + +```bash +# 1 · file drop +cp -r integrations/pi ~/.pi/agent/extensions/nemo-relay + +# 2 · pi install, from a local path or a git URL -- never with --local +pi install /path/to/NeMo-Relay/integrations/pi +``` + +`nemo-relay run --agent pi` needs neither — it passes the extension with `-e`, +which loads first and is never trust-gated. + +The extension is **not published to npm**, deliberately. `pi install` accepts a +local path or a git URL as readily as an `npm:` specifier, so publishing would +add a spelling of a route that already works rather than a capability. **Do not install into `.pi/extensions/` or with `pi install --local`.** pi adds diff --git a/integrations/pi/README.md b/integrations/pi/README.md index 96c501134..05401d15f 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -43,16 +43,31 @@ NEMO_RELAY_PI_GATEWAY_URL=http://127.0.0.1:4040 \ ### Where to install it -**User scope only.** Copy the extension into `~/.pi/agent/extensions/`, or -install it with `pi install ` *without* `--local`. +**User scope only**, by either of two routes: + +```bash +# 1 · file drop +cp -r integrations/pi ~/.pi/agent/extensions/nemo-relay + +# 2 · pi install, from a local path or a git URL -- never with --local +pi install /path/to/NeMo-Relay/integrations/pi +``` | Path | Install here? | Trust-gated? | |---|---|---| | `~/.pi/agent/extensions/` | Yes | No | -| `pi install` (user scope) | Yes | No | +| `pi install ` | Yes | No | | `-e ` | Per-invocation; what the launcher uses | No | | `.pi/extensions/` or `pi install --local` | **No** | **Yes** | +**This package is deliberately not published to npm.** `pi install` resolves a +local path or a git URL as readily as an `npm:` specifier, so publishing would +buy one more spelling of a route that already works, at the cost of an npm +namespace, a build step (the sources are TypeScript that nothing compiles today) +and release wiring. It is `private: true` for that reason, not by oversight — +though it is in the repository's version bump anyway, so the version cannot +already be stale if that ever changes. + ⚠️ **A project-scoped install is silently skipped, and nothing tells you.** pi adds project extensions to its candidate set only when the project is trusted, and `-p`, `--mode json` and `--mode rpc` never prompt for trust — so under the From 80ca784ac71e0c3d431987e01ffb7332ba988723 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 20:10:49 -0700 Subject: [PATCH 15/41] fix(pi): correct two facts a review pass found wrong Both shipped, and both were confidently stated. **`emitToolCall` is not first-wins.** The shared test harness documented, and implemented, "pi returns the first truthy result" for both gated hooks. That is true only of `emitUserBash`. `emitToolCall` runs every handler, keeps the last truthy result, and short-circuits only on `{block: true}` -- so an extension ahead of ours preempts the tool gate only by *blocking*, while any truthy result at all preempts the inline-shell gate. The driver now models each hook's own rule, and a test pins the difference. That test failed on its first run and was itself wrong in an instructive way: our allow is `undefined`, and `undefined` does not overwrite an earlier truthy result, so what pi hands back is the other extension's object -- inert, because it carries no `block`. Which is exactly why this extension returns `undefined` rather than `{}`. The comment in `index.ts` had this right all along ("returns on the first `{block: true}`"); the harness generalized it into something false. **Radius is a provider, and there are 39 of them.** The claim that pi ships 38 providers of which 6 are unserviceable came from counting the JSON files in `providers/data/`. `builtinProviders()` returns 39, and the one provider with no static catalog file is Radius -- which the earlier note then argued was "an OAuth mode, not a provider". pi's own source says otherwise: `radiusProvider()` is in the builtin list, and a comment two hundred lines up notes that purely dynamic providers such as Radius have no catalog entry. So it is 39 built-in providers, 32 serviceable, 7 unserviceable: Bedrock, Azure OpenAI Responses, Google, Google Vertex, Mistral, OpenAI Codex, and Radius (`pi-messages`). Corrected in the redirect module's comment, the extension README, the CLI docs page and the support matrix, each of which now says to count providers rather than files. The support matrix also still said 13 lifecycle hooks; it is 15. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- docs/nemo-relay-cli/pi.mdx | 12 +++++-- docs/reference/support-matrix.mdx | 2 +- integrations/pi/README.md | 7 ++-- integrations/pi/src/provider-redirect.ts | 11 ++++-- integrations/pi/test/harness.mjs | 43 ++++++++++++++++++------ integrations/pi/test/tool-call.test.mjs | 28 +++++++++++++++ 6 files changed, 83 insertions(+), 20 deletions(-) diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index 77dbe9e49..74eb41faf 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -342,9 +342,15 @@ nemo-relay --bind 127.0.0.1:4040 \ --openai-base-url https://integrate.api.nvidia.com/v1 ``` -32 of pi's 38 providers speak an API the gateway serves. The six that do not are -Amazon Bedrock, Azure OpenAI Responses, Google, Google Vertex, Mistral, and -OpenAI Codex. +32 of pi's 39 built-in providers speak an API the gateway serves. The seven that +do not are Amazon Bedrock, Azure OpenAI Responses, Google, Google Vertex, +Mistral, OpenAI Codex, and Radius. + + +The denominator is `builtinProviders()`, not the 38 catalog files in +`providers/data/`. Radius is a purely dynamic provider with no static catalog +entry, so counting files loses it. + Point pi at the gateway **root**, not the root plus `/v1`. The Anthropic SDK diff --git a/docs/reference/support-matrix.mdx b/docs/reference/support-matrix.mdx index 62b71f49d..2d51db0a7 100644 --- a/docs/reference/support-matrix.mdx +++ b/docs/reference/support-matrix.mdx @@ -66,7 +66,7 @@ and older CLI versions during installation, diagnostics, and transparent runs. | --- | --- | --- | --- | | Claude Code | 2.1.121 | Persistent install, transparent run, lifecycle hooks, local gateway routing, and pre-tool security | Claude desktop, web, and application sessions are unsupported unless they expose the same local hook and gateway controls. Optimization requires gateway-routed LLM traffic and available hooks. | | Codex CLI | 0.143.0 | Persistent install, transparent run, 10 supported plugin hooks, local gateway routing, and pre-tool security | Cloud or remote tasks that bypass the local machine have partial or no LLM capture. The plugin hook schema has no `SessionEnd`; Relay finalizes the cumulative session snapshot at `Stop`. Encrypted Codex multi-agent v2 payloads cannot be decrypted or reliably linked. | -| pi | 0.84.0 | Transparent run through a Relay-authored pi extension, 13 lifecycle hooks, tool-call security, inline-shell security, and model-call security when redirection applies | Proof of concept. No persistent install: pi has no plugin marketplace. Model traffic is redirected only when the gateway forwards to the endpoint the selected model would otherwise call; otherwise there are no LLM spans for that model. Six of pi's 38 providers speak an API the gateway has no route for. Subagents and nested pi processes appear as unrelated sessions. pi ships breaking changes through minor releases, so hook signatures need re-verification after an upgrade. | +| pi | 0.84.0 | Transparent run through a Relay-authored pi extension, 15 lifecycle hooks, tool-call security, inline-shell security, and model-call security when redirection applies | Proof of concept. No persistent install: pi has no plugin marketplace. Model traffic is redirected only when the gateway forwards to the endpoint the selected model would otherwise call; otherwise there are no LLM spans for that model. Seven of pi's 39 built-in providers speak an API the gateway has no route for. Subagents and nested pi processes appear as unrelated sessions. pi ships breaking changes through minor releases, so hook signatures need re-verification after an upgrade. | For installation, diagnostics, and host-specific behavior, refer to [Coding Agent Installation](/nemo-relay-cli/plugin-installation), [Claude diff --git a/integrations/pi/README.md b/integrations/pi/README.md index 05401d15f..5e9476fcb 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -250,7 +250,7 @@ outcome as a `model_redirect` mark so a trace without LLM spans explains itself. |---|---| | Gateway upstream equals the model's endpoint | Redirected; LLM spans appear under the turn | | Gateway forwards somewhere else | Skipped, `upstream-mismatch` | -| Model's API has no gateway route (Bedrock, Azure OpenAI Responses, Google, Google Vertex, Mistral, OpenAI Codex) | Skipped, `unserviceable-api` | +| Model's API has no gateway route (Bedrock, Azure OpenAI Responses, Google, Google Vertex, Mistral, OpenAI Codex, Radius) | Skipped, `unserviceable-api` | | Launched outside `nemo-relay run --agent pi`, so the upstream is unknown | Skipped, `unknown-upstream` — set `NEMO_RELAY_PI_REDIRECT=force` to override | `nemo-relay run --agent pi` sets the two upstream variables for you. Running pi @@ -259,7 +259,10 @@ by hand against a standalone gateway means setting them yourself, or forcing. The decision is re-evaluated on every `model_select`, so switching to a model the gateway does not front stops redirecting rather than silently misrouting. -32 of pi's 38 providers speak an API the gateway serves; the six above do not. +32 of pi's 39 built-in providers speak an API the gateway serves; the seven +above do not. Count from `builtinProviders()` rather than from the 38 files in +`providers/data/`: Radius is a purely dynamic provider with no static catalog +entry, so a file count loses it. ## Hook mapping diff --git a/integrations/pi/src/provider-redirect.ts b/integrations/pi/src/provider-redirect.ts index ef9307f4e..45e8a3be8 100644 --- a/integrations/pi/src/provider-redirect.ts +++ b/integrations/pi/src/provider-redirect.ts @@ -144,9 +144,14 @@ export function decideRedirect( const family = SERVICEABLE_APIS[model.api]; if (!family) { - // Six of pi's providers speak an API the gateway has no route for - // (Bedrock, Azure OpenAI Responses, Google, Google Vertex, Mistral, OpenAI - // Codex). Redirecting them would 404 rather than degrade. + // Seven of pi's 39 built-in providers speak an API the gateway has no + // route for: Bedrock, Azure OpenAI Responses, Google, Google Vertex, + // Mistral, OpenAI Codex, and Radius (`pi-messages`). Redirecting them would + // 404 rather than degrade. + // + // Counted from `builtinProviders()`, not from the 38 files in + // `providers/data/` -- Radius is a purely dynamic provider with no static + // catalog entry, so a file count silently loses it. return { kind: 'skip', code: 'unserviceable-api', diff --git a/integrations/pi/test/harness.mjs b/integrations/pi/test/harness.mjs index 75bee6172..a0b690b68 100644 --- a/integrations/pi/test/harness.mjs +++ b/integrations/pi/test/harness.mjs @@ -4,12 +4,20 @@ /** * Shared test harness: a stub gateway, and a driver that fires pi's hooks. * - * ⚠️ **The driver returns the *first* non-`undefined` result, not the last.** - * That is pi's rule, in both `emitToolCall` and `emitUserBash`, and it is the - * difference between "our gate decided" and "an extension ahead of us decided - * and we never ran". A last-wins driver silently inverts the trap this - * extension documents in two places, and would let a regression that breaks - * preemption behaviour pass. + * ⚠️ **The two gated hooks resolve competing handlers by different rules, and + * the driver has to model each one.** A driver that picks either rule for both + * misrepresents one of them, and preemption is exactly what these tests exist + * to pin. + * + * | Hook | pi's rule | Catches? | + * |---|---|---| + * | `tool_call` | Runs **every** handler, keeps the **last** truthy result, and returns early **only** on `{block: true}` | No — an exception propagates | + * | `user_bash` | Returns the **first** truthy result and stops | Yes — a throw fails *open* | + * + * So an earlier extension preempts the tool gate only by *blocking*; a + * non-blocking result from one does not stop us, it is simply overwritten by + * whichever handler answers last. On the inline-shell path any truthy result at + * all preempts, which is why `{}` is dangerous there. */ import { createServer } from 'node:http'; @@ -95,14 +103,27 @@ export function load(extension, options = {}) { ...options.ctx, }; return async (name, event = {}) => { + let last; for (const handler of handlers.get(name) ?? []) { const result = await handler({ type: name, ...event }, ctx); - // First truthy result wins and stops iteration -- pi's rule. Note that - // `{}` is truthy: an allow must be `undefined`, or it silently preempts - // every extension behind it while deciding nothing. - if (result !== undefined) return result; + if (!result) continue; + if (name === 'user_bash') { + // First truthy result wins and stops iteration. `{}` is truthy, so an + // allow must be `undefined` or it silently preempts every extension + // behind it while deciding nothing. + return result; + } + if (name === 'tool_call') { + // Every handler runs; only a block short-circuits. A non-blocking + // truthy result does not preempt -- it is overwritten by whoever + // answers last, which is why returning one is inert rather than fatal. + last = result; + if (result.block) return result; + continue; + } + last = result; } - return undefined; + return last; }; } diff --git a/integrations/pi/test/tool-call.test.mjs b/integrations/pi/test/tool-call.test.mjs index efe463400..04f5519f7 100644 --- a/integrations/pi/test/tool-call.test.mjs +++ b/integrations/pi/test/tool-call.test.mjs @@ -201,6 +201,34 @@ describe('an extension ahead of the gate', () => { ); }); + // The two hooks do NOT resolve competing handlers the same way, and an earlier + // version of this harness papered over the difference. `emitToolCall` runs every + // handler and short-circuits only on a block, so a non-blocking result from an + // extension ahead of us is inert -- our gate still runs, and the gateway still + // sees the call. Only `emitUserBash` is first-truthy-wins. + it('does not preempt the tool gate with a non-blocking result', async () => { + const fire = load(extension, { + // Truthy, but no `block`: pi keeps going. + before: { tool_call: async () => ({ reason: 'just an opinion' }) }, + }); + + const result = await fire('tool_call', call()); + + // What pi hands back is the earlier extension's object, not ours: our allow + // is `undefined`, and `undefined` does not overwrite a previous truthy + // result. It is inert because it carries no `block` -- which is the whole + // reason this extension returns `undefined` rather than `{}` on an allow. + assert.deepEqual(result, { reason: 'just an opinion' }); + assert.notEqual(result.block, true, 'inert: nothing was blocked'); + + await drain(fire); + assert.equal( + named(gateway.posts, 'tool_call').length, + 1, + 'and our gate still ran, so the gateway saw and decided the call', + ); + }); + it('preempts the inline-shell gate the same way', async () => { const fire = load(extension, { before: { From f2354af320685d983a922d84545066074be22ac0 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 20:14:53 -0700 Subject: [PATCH 16/41] docs(pi): drop a backwards reason from the inline-shell notes The claim that a rewritten inline command cannot be executed because "the terminal component has already been built from it" is backwards: pi constructs `BashExecutionComponent` *after* `emitUserBash` resolves, which is precisely why a slow gateway shows a frozen prompt -- a limitation these same docs record two paragraphs away. The conclusion is unaffected and rests on the other half of the sentence: both call sites pass the original command straight to `executeBash` and read nothing back out of the event, so there is nowhere to put a rewrite. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- integrations/pi/README.md | 4 ++-- integrations/pi/src/user-bash.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/integrations/pi/README.md b/integrations/pi/README.md index 5e9476fcb..b6b814495 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -197,8 +197,8 @@ says explicitly that it is an infrastructure fault rather than a judgement. **A rewritten command is refused, not run.** pi's `user_bash` result can replace the *result* or the execution backend, but never the command — both call sites -pass the original text on, and the terminal component has already been built -from it. So a request intercept that rewrites an inline command cannot be +pass the original text straight on to `executeBash` and read nothing back out of +the event. So a request intercept that rewrites an inline command cannot be honoured, and the command is refused rather than run unmodified, on the same rule the tool path applies to a transform it cannot apply safely. diff --git a/integrations/pi/src/user-bash.ts b/integrations/pi/src/user-bash.ts index 90e63c9d8..435d52512 100644 --- a/integrations/pi/src/user-bash.ts +++ b/integrations/pi/src/user-bash.ts @@ -98,8 +98,8 @@ export function refusalResult(reason: string): BashResult { * * pi's `UserBashEventResult` can replace the *result* or supply custom * execution `operations`, but it cannot replace the command: both call sites - * pass the original string on to `executeBash`, and the terminal component has - * already been built from it. Taking over execution to run the rewrite instead + * pass the original string on to `executeBash`, and neither reads anything back + * out of the event. Taking over execution to run the rewrite instead * would mean reimplementing pi's shell selection, command prefix and * process-tree cancellation, which is a behaviour change the sidecar has no * business making. From 4c5f63a11bcc2113cc66dc7ed27015d42db5a268 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 22:15:22 -0700 Subject: [PATCH 17/41] fix(pi): close four review findings, two of them fail-open **An unreadable 2xx was a silent allow.** A success body may carry a required argument transform, so a body that does not parse, or parses to something that is not an object, cannot be treated as an empty allow -- doing so runs the original arguments and discards the policy, which is the exact failure a refused transform blocks the call to prevent. It is now a fault, resolved through `NEMO_RELAY_PI_FAIL` like any other. **A malformed transform id was accepted.** The check was `typeof id === 'string' && id !== toolCallId`, so a missing, numeric or object id skipped it entirely and the transform applied. The echoed id is the only thing proving the rewrite belongs to the call just posted; it now has to be a string and exactly equal. **`nemo-relay run -- pi` did not work.** pi was added to `CodingAgent` without being added to executable inference, so the bare-name form failed while the identical Claude and Codex forms worked. One arm, and it inherits the existing basename and `.exe`/`.cmd` handling; the test covers paths, case and suffixes, and pins that `pip`/`pipx`/`pi-sbx` are not pi. **`nemo-relay config` deleted `agents.pi`.** Two halves, both needed: the wizard offered a hard-coded Claude/Codex pair, and `read_agents_from_doc` hand-matched the same two keys. An unscoped save replaces the whole `[agents]` table with what those two produced, so an agent neither could name was silently dropped from the user's configuration. Both now derive from `CodingAgent::ALL`, so adding an agent cannot reintroduce this. A fifth finding in the same review -- that the shared test harness models `emitToolCall` as first-truthy-wins -- was already fixed in `80ca784a`. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- crates/cli/src/agents/mod.rs | 1 + crates/cli/src/commands/configure/model.rs | 14 +++-- .../src/commands/configure/wizard/prompt.rs | 5 +- .../tests/coverage/agents/launcher_tests.rs | 24 ++++++++ .../cli/tests/coverage/shared/setup_tests.rs | 56 +++++++++++++++++++ integrations/pi/src/argument-transform.ts | 13 +++-- integrations/pi/src/gateway-client.ts | 12 +++- .../pi/test/argument-transform.test.mjs | 20 +++++++ integrations/pi/test/gateway-client.test.mjs | 19 ++++++- 9 files changed, 148 insertions(+), 16 deletions(-) diff --git a/crates/cli/src/agents/mod.rs b/crates/cli/src/agents/mod.rs index 5dcb7b9a6..9b03f024d 100644 --- a/crates/cli/src/agents/mod.rs +++ b/crates/cli/src/agents/mod.rs @@ -154,6 +154,7 @@ impl CodingAgent { match name { "claude" | "claude-code" => Some(Self::ClaudeCode), "codex" => Some(Self::Codex), + "pi" => Some(Self::Pi), _ => None, } } diff --git a/crates/cli/src/commands/configure/model.rs b/crates/cli/src/commands/configure/model.rs index 396d72e1e..f62634b7d 100644 --- a/crates/cli/src/commands/configure/model.rs +++ b/crates/cli/src/commands/configure/model.rs @@ -280,14 +280,16 @@ pub(crate) fn read_agents_from_doc(doc: &DocumentMut) -> Vec { let Some(table) = doc.get("agents").and_then(|i| i.as_table()) else { return Vec::new(); }; + // Driven from `CodingAgent::ALL` rather than a hand-written match: an agent missing + // from this list is not just unreported, it is *deleted*. An unscoped wizard run + // rewrites the whole `[agents]` table from what it read here, so a key it cannot + // parse silently disappears from the user's configuration. let mut found = Vec::new(); for (key, _) in table.iter() { - let agent = match key { - "claude" => Some(CodingAgent::ClaudeCode), - "codex" => Some(CodingAgent::Codex), - _ => None, - }; - if let Some(agent) = agent { + if let Some(agent) = CodingAgent::ALL + .into_iter() + .find(|agent| agent_key_and_command(*agent).0 == key) + { found.push(agent); } } diff --git a/crates/cli/src/commands/configure/wizard/prompt.rs b/crates/cli/src/commands/configure/wizard/prompt.rs index 0d9f8ff62..10379d896 100644 --- a/crates/cli/src/commands/configure/wizard/prompt.rs +++ b/crates/cli/src/commands/configure/wizard/prompt.rs @@ -197,7 +197,10 @@ fn ask_agents( detected: &[CodingAgent], configured: &[CodingAgent], ) -> Result, CliError> { - let all_supported = [CodingAgent::ClaudeCode, CodingAgent::Codex]; + // Every supported agent, not a hand-written pair. Saving an unscoped result replaces + // the whole `[agents]` table, so an agent missing from this list is removed from the + // user's config by a wizard that never offered it. + let all_supported = CodingAgent::ALL; let labels: Vec = all_supported .iter() .map(|a| { diff --git a/crates/cli/tests/coverage/agents/launcher_tests.rs b/crates/cli/tests/coverage/agents/launcher_tests.rs index 6deb4e2d5..fdf949787 100644 --- a/crates/cli/tests/coverage/agents/launcher_tests.rs +++ b/crates/cli/tests/coverage/agents/launcher_tests.rs @@ -98,6 +98,30 @@ fn infers_agent_from_command_or_uses_override() { assert_eq!(agent, CodingAgent::ClaudeCode); } +// pi was added to `CodingAgent` without being added here, so `nemo-relay run -- pi` +// failed while the identical Claude and Codex forms worked -- and the error named +// only those two, so nothing pointed at the cause. +#[test] +fn infers_pi_from_a_bare_name_a_path_or_a_windows_suffix() { + for command in [ + "pi", + "/usr/local/bin/pi", + "PI", + "pi.exe", + r"C:\tools\pi.cmd", + ] { + assert_eq!( + CodingAgent::infer(command), + Some(CodingAgent::Pi), + "should infer pi from {command}" + ); + } + // Not a false positive magnet: only the exact basename counts. + for command in ["pip", "pipx", "mypi", "pi-sbx"] { + assert_eq!(CodingAgent::infer(command), None, "{command} is not pi"); + } +} + #[test] fn uses_configured_command_when_no_argv_is_supplied() { let agents = AgentConfigs { diff --git a/crates/cli/tests/coverage/shared/setup_tests.rs b/crates/cli/tests/coverage/shared/setup_tests.rs index 57bc7c58f..0900c58c9 100644 --- a/crates/cli/tests/coverage/shared/setup_tests.rs +++ b/crates/cli/tests/coverage/shared/setup_tests.rs @@ -284,6 +284,62 @@ command = "custom" assert_eq!(agents, vec![CodingAgent::ClaudeCode, CodingAgent::Codex]); } +// The parser used to hand-match "claude" and "codex", so `[agents.pi]` read as +// unknown. That is not merely unreported: an unscoped wizard run rewrites the whole +// `[agents]` table from what this returns, so an unparsed agent is *deleted*. +#[test] +fn read_agents_from_doc_recognizes_every_supported_agent() { + let doc: DocumentMut = r#" +[agents.claude] +command = "claude" + +[agents.codex] +command = "codex" + +[agents.pi] +command = "pi" +"# + .parse() + .unwrap(); + assert_eq!( + read_agents_from_doc(&doc), + vec![CodingAgent::ClaudeCode, CodingAgent::Codex, CodingAgent::Pi] + ); +} + +// The failure this guards: `nemo-relay config`, answered without deselecting anything, +// silently dropping an agent the wizard never offered. +#[test] +fn an_unscoped_save_preserves_every_configured_agent() { + let home = tempfile::tempdir().unwrap(); + let path = home.path().join(".config/nemo-relay/config.toml"); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write( + &path, + "[agents.claude]\ncommand = \"claude\"\n\n[agents.pi]\ncommand = \"pi\"\n", + ) + .unwrap(); + + // What the wizard reads back as its pre-checked defaults... + let existing: DocumentMut = std::fs::read_to_string(&path).unwrap().parse().unwrap(); + let configured = read_agents_from_doc(&existing); + assert!( + configured.contains(&CodingAgent::Pi), + "pi must be seen as configured, or the wizard cannot re-offer it: {configured:?}" + ); + + // ...and what it writes back when the user changes nothing. + let doc = build_config(&SetupAnswers { agents: configured }); + save_config(&doc, home.path(), None).unwrap(); + + let saved: DocumentMut = std::fs::read_to_string(&path).unwrap().parse().unwrap(); + assert_eq!( + read_agents_from_doc(&saved), + vec![CodingAgent::ClaudeCode, CodingAgent::Pi], + "an unscoped save replaces the whole [agents] table, so it must carry every agent forward" + ); +} + #[test] fn read_existing_defaults_reads_user_config_and_ignores_project_config() { let cwd = tempfile::tempdir().unwrap(); diff --git a/integrations/pi/src/argument-transform.ts b/integrations/pi/src/argument-transform.ts index 23ef2bcf9..9e52de4fc 100644 --- a/integrations/pi/src/argument-transform.ts +++ b/integrations/pi/src/argument-transform.ts @@ -112,12 +112,17 @@ export function decideTransform( const envelope = body?.tool_call; if (!envelope || envelope.input === undefined) return { kind: 'none' }; - // A body for a different call means the gateway and the extension disagree about what is in - // flight. Applying it would rewrite one tool call with another's arguments. - if (typeof envelope.tool_call_id === 'string' && envelope.tool_call_id !== toolCallId) { + // The echoed id is what proves the transform belongs to the call we just posted, so it has to be + // present and exact. Accepting a missing or non-string id would let a truncated or malformed body + // rewrite the wrong call's arguments -- the failure the echo exists to prevent. + if (typeof envelope.tool_call_id !== 'string' || envelope.tool_call_id !== toolCallId) { + const named = + typeof envelope.tool_call_id === 'string' + ? envelope.tool_call_id + : JSON.stringify(envelope.tool_call_id) ?? 'nothing'; return { kind: 'refuse', - reason: `the transform names tool call ${envelope.tool_call_id}, not ${toolCallId}`, + reason: `the transform names tool call ${named}, not ${toolCallId}`, }; } diff --git a/integrations/pi/src/gateway-client.ts b/integrations/pi/src/gateway-client.ts index b5b8ca4f1..5dceb4246 100644 --- a/integrations/pi/src/gateway-client.ts +++ b/integrations/pi/src/gateway-client.ts @@ -84,10 +84,16 @@ export async function postHook( }); if (response.ok) { - // An allow body is `{}` unless a request intercept rewrote the arguments, so parsing is - // best effort: a body we cannot read is still an allow, just one with nothing to apply. + // An allow body is `{}` unless a request intercept rewrote the arguments -- so a body we + // cannot read is NOT a plain allow. It may have carried a transform, and treating it as an + // empty allow would run the original arguments while silently discarding a policy decision: + // exactly the failure a refused transform blocks the call to prevent. An unreadable success + // is an infrastructure fault, resolved by `NEMO_RELAY_PI_FAIL` like any other. const body = await safeJson(response); - return body && typeof body === 'object' ? { kind: 'allow', body } : { kind: 'allow' }; + if (body === null || typeof body !== 'object' || Array.isArray(body)) { + return { kind: 'fault', detail: 'gateway returned a success body that is not a JSON object' }; + } + return { kind: 'allow', body }; } if (response.status === 403) { diff --git a/integrations/pi/test/argument-transform.test.mjs b/integrations/pi/test/argument-transform.test.mjs index 6b686879b..c76df084b 100644 --- a/integrations/pi/test/argument-transform.test.mjs +++ b/integrations/pi/test/argument-transform.test.mjs @@ -84,6 +84,26 @@ describe('transform decision', () => { assert.match(outcome.reason, /call-2, not call-1/); }); + // The echoed id is the only thing proving the transform belongs to the call we + // posted. A missing or non-string one used to skip the check entirely, so a + // truncated body could rewrite the wrong call's arguments. + it('refuses a transform whose call id is missing or not a string', () => { + // Built by hand rather than through `envelope`, whose default parameter would + // substitute a valid id for the absent case and quietly test nothing. + const bodies = [ + ['absent', { tool_call: { input: { path: 'b.txt' } } }], + ['null', { tool_call: { tool_call_id: null, input: { path: 'b.txt' } } }], + ['number', { tool_call: { tool_call_id: 42, input: { path: 'b.txt' } } }], + ['object', { tool_call: { tool_call_id: { id: CALL }, input: { path: 'b.txt' } } }], + ['array', { tool_call: { tool_call_id: [CALL], input: { path: 'b.txt' } } }], + ]; + for (const [label, body] of bodies) { + const outcome = decideTransform(body, CALL, { path: 'a.txt' }); + assert.equal(outcome.kind, 'refuse', `a ${label} call id must be refused`); + assert.match(outcome.reason, /not call-1/); + } + }); + it('refuses a non-object transform', () => { const outcome = decideTransform(envelope('rm -rf /'), CALL, { path: 'a.txt' }); assert.equal(outcome.kind, 'refuse'); diff --git a/integrations/pi/test/gateway-client.test.mjs b/integrations/pi/test/gateway-client.test.mjs index d512931ee..6a6e4e531 100644 --- a/integrations/pi/test/gateway-client.test.mjs +++ b/integrations/pi/test/gateway-client.test.mjs @@ -29,10 +29,11 @@ function serve(handler) { }); req.on('end', () => { received.push({ url: req.url, headers: req.headers, body: JSON.parse(body || '{}') }); - const { status, payload, delayMs } = handler(received.at(-1)); + const { status, payload, raw, delayMs } = handler(received.at(-1)); const send = () => { res.writeHead(status, { 'content-type': 'application/json' }); - res.end(JSON.stringify(payload ?? {})); + // `raw` lets a case emit a body JSON.parse cannot read. + res.end(raw ?? JSON.stringify(payload ?? {})); }; if (delayMs) setTimeout(send, delayMs); else send(); @@ -62,6 +63,9 @@ describe('gateway client wire contract', () => { ctx = serve((request) => { const name = request.body.hook_event_name; if (name === 'slow') return { status: 200, payload: {}, delayMs: 500 }; + if (name === 'bad-json') return { status: 200, raw: '{ truncated' }; + if (name === 'array-body') return { status: 200, payload: [] }; + if (name === 'string-body') return { status: 200, payload: 'ok' }; if (name === 'boom') return { status: 500, payload: { error: { message: 'kaboom' } } }; if (name === 'naked-403') return { status: 403, payload: { error: { message: 'nope' } } }; if (request.body.tool_name === 'read' && request.body.input?.path?.endsWith('.env')) { @@ -84,6 +88,17 @@ describe('gateway client wire contract', () => { after(() => ctx.server.close()); + // A 2xx body may carry a required argument transform, so an unreadable one is not + // an empty allow -- treating it as one runs the original arguments and silently + // discards a policy decision, which is the failure a refused transform blocks for. + it('treats an unreadable or non-object 2xx body as a fault, not a bare allow', async () => { + for (const name of ['bad-json', 'array-body', 'string-body']) { + const outcome = await postHook(baseConfig(url), { hook_event_name: name }); + assert.equal(outcome.kind, 'fault', `${name} must not be a plain allow`); + assert.match(outcome.detail, /not a JSON object/); + } + }); + it('treats 2xx as allow', async () => { const outcome = await postHook(baseConfig(url), { hook_event_name: 'tool_call', From b7c8f464f5207056159af16777b4ba546fe55f1d Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 22:28:13 -0700 Subject: [PATCH 18/41] fix(pi): close the remaining review findings **A redirect decided from one model moved its siblings.** `registerProvider(name, {baseUrl})` rewrites every model of a provider, so judging safety from the selected model alone is judging on its siblings' behalf. Several pi 0.84 providers mix API families at different paths -- Fireworks serves anthropic-messages at `/inference` and openai-completions at `/inference/v1` -- so picking a Fireworks Anthropic model pointed its OpenAI siblings at api.openai.com, carrying a Fireworks key and a Fireworks model id. Mid-session, after the user changed only the model, with no mark explaining it, because the `already-redirected` short-circuit suppressed re-evaluation. `decideRedirect` now takes the provider's catalog as it was *before* any registration -- snapshotted once, never refreshed, since after registering every model reports the gateway's URL and the check would compare the gateway against itself -- and skips with `provider-mixed-endpoints` naming the sibling that fails. Pointing both upstreams at the provider makes all of them pass. **The doctor could not see the install it recommends.** `pi install` writes nothing into the extension directories; it appends the source to a `packages` array in `settings.json`, user scope at `/settings.json` and `--local` at `/.pi/settings.json`. Scanning only the directories told a user who had just run the recommended command that no extension was found, and could not see a trust-gated `--local` entry at all -- the case the module exists for. It reads both settings files now. The old rationale comment was also simply wrong: it claimed `pi install` "renames and nests what it writes". **The advertised git install never worked.** pi has no subdirectory syntax for a git source: it clones the repository root and looks there for a `pi` manifest key or a top-level `extensions/`. This extension is at `integrations/pi`, so the install reported success and loaded nothing -- while the clone's root `skills/` *was* picked up, giving pi our Codex and Claude skills and none of the gating. Removed from both docs. The same paragraph's npm reasoning rested on the same mistake, so it is restated: the file drop and the local path are the two working routes, and `integrations/pi/package.json` now carries a `pi.extensions` key so both resolve explicitly rather than through pi's directory fallback. **Redirected model calls carried no session key.** Hook posts send `x-nemo-relay-session-id`; provider requests are the other stream and sent nothing, so with two pi sessions on one gateway the LLM spans were split off into an isolated root. A `before_provider_headers` handler injects it -- gated on the redirected set, because the hook is global and an internal session id must not leak to a provider we deliberately did not redirect, and reading the id live because the cached config would go stale across a session replacement. **Two smaller ones.** The inline-shell span closed as `ok` before pi had run the command; it records `policy-allowed` now, since pi reports no completion and the gate knows what it decided, never what happened. And the `model_redirect` mark dropped the stable skip `code`, leaving consumers to pattern-match prose. **One documented rather than fixed.** pi runs every `tool_call` handler unless one blocks, sharing one mutable `input` with no re-validation, so an extension loaded *after* this gate can rewrite arguments Relay authorized. pi offers no ordering API and no post-chain hook, so it cannot be prevented from inside an extension. Both docs now say the gate is authoritative over the model, not over the other extensions. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- crates/cli/src/agents/pi/doctor.rs | 84 +++++++++++++++---- .../tests/coverage/agents/pi_doctor_tests.rs | 71 ++++++++++++++++ docs/nemo-relay-cli/pi.mdx | 31 ++++++- integrations/pi/README.md | 43 +++++++--- integrations/pi/index.ts | 63 +++++++++++++- integrations/pi/package.json | 5 ++ integrations/pi/src/pi-hook-types.ts | 35 +++++++- integrations/pi/src/provider-redirect.ts | 46 +++++++++- .../pi/test/provider-redirect.test.mjs | 63 ++++++++++++++ integrations/pi/test/user-bash.test.mjs | 4 +- 10 files changed, 411 insertions(+), 34 deletions(-) diff --git a/crates/cli/src/agents/pi/doctor.rs b/crates/cli/src/agents/pi/doctor.rs index 27ab98e72..012fdb105 100644 --- a/crates/cli/src/agents/pi/doctor.rs +++ b/crates/cli/src/agents/pi/doctor.rs @@ -31,6 +31,9 @@ const PI_AGENT_DIR_ENV: &str = "PI_CODING_AGENT_DIR"; /// pi's configuration directory name, from its `piConfig.configDir`. const PI_CONFIG_DIR: &str = ".pi"; +/// Where pi records installed packages, in both scopes. +const PI_SETTINGS_FILE: &str = "settings.json"; + /// Gateway URL the extension falls back to when nothing else resolves one. /// Kept in step with `configFromEnv` in `integrations/pi/src/gateway-client.ts`. const DEFAULT_GATEWAY_URL: &str = "http://127.0.0.1:4040"; @@ -109,11 +112,19 @@ pub(crate) fn gateway_url(bind: Option) -> String { /// Every place pi could load an extension from, that currently holds one. /// -/// Deliberately reports *any* auto-discovered entry rather than trying to -/// recognize the NeMo Relay extension by filename: `pi install` renames and -/// nests what it writes, so a filename match would miss the installed layout -/// and quietly report nothing -- which is the failure mode being guarded -/// against. The trust question is a property of the directory, not of the file. +/// **Two unrelated routes, and missing either one makes this check lie.** +/// +/// *Auto-discovery* reads `/extensions` and `/.pi/extensions`. +/// Any entry counts; the trust question is a property of the directory, not of +/// the file, so there is nothing to recognize by name. +/// +/// *`pi install`* does not touch those directories at all. It appends the source +/// to a `packages` array in `settings.json` -- `/settings.json` for +/// user scope, `/.pi/settings.json` for `--local` -- and for a local path +/// copies nothing whatsoever. Scanning only the extension directories therefore +/// reported "no pi extension found" to a user who had just run the install +/// command the docs recommend, and could not see a trust-gated `--local` entry +/// at all, which is the case this whole module exists to catch. pub(crate) fn extension_sites(cwd: &Path) -> Vec { let mut sites = Vec::new(); if let Some(path) = extension_location() { @@ -130,6 +141,14 @@ pub(crate) fn extension_sites(cwd: &Path) -> Vec { scope: ExtensionScope::User, }); } + if let Some(path) = user_settings_path() + && settings_declare_packages(&path) + { + sites.push(ExtensionSite { + path, + scope: ExtensionScope::User, + }); + } let project_dir = cwd.join(PI_CONFIG_DIR).join("extensions"); if directory_has_entries(&project_dir) { sites.push(ExtensionSite { @@ -137,19 +156,56 @@ pub(crate) fn extension_sites(cwd: &Path) -> Vec { scope: ExtensionScope::Project, }); } + let project_settings = cwd.join(PI_CONFIG_DIR).join(PI_SETTINGS_FILE); + if settings_declare_packages(&project_settings) { + sites.push(ExtensionSite { + path: project_settings, + scope: ExtensionScope::Project, + }); + } sites } -/// `~/.pi/agent/extensions`, honoring pi's own directory override. +/// `/settings.json`, where `pi install` records a user-scope package. +fn user_settings_path() -> Option { + Some(pi_agent_dir()?.join(PI_SETTINGS_FILE)) +} + +/// Whether a pi settings file declares at least one installed package. +/// +/// Deliberately tolerant: an unreadable or malformed settings file is reported as +/// "no packages" rather than as an error. This check exists to find something the +/// user installed, and a parse failure here is pi's problem to report, not a +/// reason for `doctor` to fail. +fn settings_declare_packages(path: &Path) -> bool { + std::fs::read_to_string(path) + .ok() + .and_then(|raw| serde_json::from_str::(&raw).ok()) + .and_then(|settings| { + settings + .get("packages") + .and_then(serde_json::Value::as_array) + .map(|packages| !packages.is_empty()) + }) + .unwrap_or(false) +} + +/// `~/.pi/agent`, honoring pi's own directory override. +fn pi_agent_dir() -> Option { + match std::env::var_os(PI_AGENT_DIR_ENV) { + Some(dir) if !dir.is_empty() => Some(PathBuf::from(dir)), + _ => Some( + crate::agents::shared::host::home_dir() + .ok()? + .join(PI_CONFIG_DIR) + .join("agent"), + ), + } +} + +/// `~/.pi/agent/extensions`, the auto-discovery directory. fn user_extensions_dir() -> Option { - let agent_dir = match std::env::var_os(PI_AGENT_DIR_ENV) { - Some(dir) if !dir.is_empty() => PathBuf::from(dir), - _ => crate::agents::shared::host::home_dir() - .ok()? - .join(PI_CONFIG_DIR) - .join("agent"), - }; - Some(agent_dir.join("extensions")) + Some(pi_agent_dir()?.join("extensions")) } fn directory_has_entries(path: &Path) -> bool { diff --git a/crates/cli/tests/coverage/agents/pi_doctor_tests.rs b/crates/cli/tests/coverage/agents/pi_doctor_tests.rs index 14de4cd5c..5ef8e8b7d 100644 --- a/crates/cli/tests/coverage/agents/pi_doctor_tests.rs +++ b/crates/cli/tests/coverage/agents/pi_doctor_tests.rs @@ -104,6 +104,77 @@ fn an_explicit_path_that_does_not_exist_is_not_reported() { assert!(!extension_configured()); } +// `pi install` writes nothing into the extension directories -- it appends the +// source to a `packages` array in settings.json. Scanning only the directories +// therefore told a user who had just run the *recommended* install command that +// no extension was found. +#[test] +fn a_pi_install_at_user_scope_is_found_in_settings_not_in_a_directory() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + std::fs::write( + agent_dir.join("settings.json"), + r#"{"packages": ["../../../NeMo-Relay/integrations/pi"]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = extension_sites(temp.path()); + + assert_eq!(sites.len(), 1, "{sites:?}"); + assert_eq!(sites[0].scope, ExtensionScope::User); +} + +// The dangerous half: `pi install --local` records the package in the project's +// settings, which is trust-gated exactly like `.pi/extensions`. Before this, the +// check could not see it at all. +#[test] +fn a_local_pi_install_is_reported_as_project_scoped() { + let temp = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(temp.path().join(".pi")).unwrap(); + std::fs::write( + temp.path().join(".pi").join("settings.json"), + r#"{"packages": ["../extensions/nemo-relay"]}"#, + ) + .unwrap(); + let empty_home = temp.path().join("home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let _env = scoped(None, Some(empty_home.as_os_str())); + let sites = extension_sites(temp.path()); + + assert!( + sites + .iter() + .any(|site| site.scope == ExtensionScope::Project), + "a --local install is trust-gated and must be reported: {sites:?}" + ); +} + +#[test] +fn settings_without_packages_are_not_a_finding() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + let _env = scoped(None, Some(agent_dir.as_os_str())); + + // Every shape pi can leave behind that means "nothing installed", plus a + // malformed file: a parse failure is pi's to report, not doctor's to fail on. + for body in [ + r#"{}"#, + r#"{"packages": []}"#, + r#"{"packages": "not-an-array"}"#, + "{ truncated", + ] { + std::fs::write(agent_dir.join("settings.json"), body).unwrap(); + assert!( + extension_sites(temp.path()).is_empty(), + "settings body {body} must not be reported as an install" + ); + } +} + #[test] fn the_gateway_url_matches_what_the_extension_resolves() { let _env = EnvScope::set(&[(PI_GATEWAY_URL_ENV, None)]); diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index 74eb41faf..568b206a8 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -68,16 +68,25 @@ Two routes, both untrusted-project-proof: # 1 · file drop cp -r integrations/pi ~/.pi/agent/extensions/nemo-relay -# 2 · pi install, from a local path or a git URL -- never with --local +# 2 · pi install, from a LOCAL PATH -- never with --local pi install /path/to/NeMo-Relay/integrations/pi ``` `nemo-relay run --agent pi` needs neither — it passes the extension with `-e`, which loads first and is never trust-gated. -The extension is **not published to npm**, deliberately. `pi install` accepts a -local path or a git URL as readily as an `npm:` specifier, so publishing would -add a spelling of a route that already works rather than a capability. + +**Do not install from a git URL.** pi has no subdirectory syntax for a git +source: it clones the repository root and looks there for a `pi` manifest key or +a top-level `extensions/` directory. This extension lives at `integrations/pi`, +so the install reports success and loads nothing — while the clone's root +`skills/` directory *is* picked up, giving you NeMo Relay's skills in pi and none +of the gating. + + +The extension is **not published to npm**, deliberately: the file drop and the +local-path install already cover user scope, and publishing would add a namespace, +a build step and release wiring for a third spelling of the same thing. **Do not install into `.pi/extensions/` or with `pi install --local`.** pi adds @@ -267,6 +276,20 @@ before the span opens, but pi fires `tool_execution_end` for blocked calls too, and the gateway synthesizes the missing pair, tagged `tool_correlation_status: "agent_fallback"`. +## What The Gate Is Authoritative Over + +pi runs every `tool_call` handler unless one returns `block`, and all of them +share the same mutable `input` object with no re-validation before execution. +Loading with `-e` puts this gate first, which prevents an *earlier* extension +pre-empting it — but an extension loaded after it can rewrite arguments Relay has +already authorized, and those execute unreviewed. + + +pi exposes no ordering API and no post-chain hook, so this cannot be prevented +from inside an extension. In a mixed extension stack, the tool gate is +authoritative over **the model**, not over **the other extensions**. + + ## Tool-Result Policy Is Not Available Relay's only tool middleware that can change what a tool *returned* is the diff --git a/integrations/pi/README.md b/integrations/pi/README.md index b6b814495..4106ca223 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -49,24 +49,34 @@ NEMO_RELAY_PI_GATEWAY_URL=http://127.0.0.1:4040 \ # 1 · file drop cp -r integrations/pi ~/.pi/agent/extensions/nemo-relay -# 2 · pi install, from a local path or a git URL -- never with --local +# 2 · pi install, from a LOCAL PATH -- never with --local pi install /path/to/NeMo-Relay/integrations/pi ``` | Path | Install here? | Trust-gated? | |---|---|---| | `~/.pi/agent/extensions/` | Yes | No | -| `pi install ` | Yes | No | +| `pi install ` | Yes | No | | `-e ` | Per-invocation; what the launcher uses | No | | `.pi/extensions/` or `pi install --local` | **No** | **Yes** | - -**This package is deliberately not published to npm.** `pi install` resolves a -local path or a git URL as readily as an `npm:` specifier, so publishing would -buy one more spelling of a route that already works, at the cost of an npm -namespace, a build step (the sources are TypeScript that nothing compiles today) -and release wiring. It is `private: true` for that reason, not by oversight — -though it is in the repository's version bump anyway, so the version cannot -already be stale if that ever changes. +| `pi install ` | **No** — see below | — | + +⚠️ **A git URL does not work, and fails silently.** pi has no +subdirectory syntax for a git source: it clones the repository *root*, then looks +there for a `pi` key in `package.json` or a top-level `extensions/` directory. +This extension lives at `integrations/pi`, and the repository root has neither — +so `pi install https://github.com/NVIDIA/NeMo-Relay` reports success, prints an +install path, and loads no extension. Worse than nothing: the clone's root +`skills/` *is* picked up, so you get NeMo Relay's Codex and Claude skills in pi +and none of the gating. + +**This package is deliberately not published to npm.** The file drop and the +local-path install both work and cover user scope; publishing would add an npm +namespace, a build step (the sources are TypeScript that nothing compiles today), +a `files` allowlist and release wiring for a third spelling. It is +`private: true` for that reason, not by oversight — and it carries a +`pi.extensions` manifest key so both working routes resolve explicitly rather +than relying on pi's directory fallback. ⚠️ **A project-scoped install is silently skipped, and nothing tells you.** pi adds project extensions to its candidate set only when the project is trusted, @@ -345,6 +355,19 @@ worth stating separately: a child pi process running this extension resolves its *own* session id and posts under it, so it does not appear as a subagent of the parent. It appears as an unrelated session. +**An authoritative boundary against *later* extensions.** pi runs every +`tool_call` handler unless one returns `block`, and they all share the same +mutable `input` object with no re-validation afterwards. Loaded with `-e` this +gate runs **first**, which is what stops an earlier extension pre-empting it — +but it also means an extension loaded *after* it can rewrite the arguments once +Relay has authorized them, and those arguments execute unreviewed. + +pi offers no ordering API and no post-chain hook, so this cannot be prevented +from inside the extension. **In a mixed extension stack, treat the tool gate as +authoritative over the model, not over the other extensions.** It is sound when +this is the only extension mutating tool arguments, which is the deployment +`nemo-relay run --agent pi` produces. + **Tool-result policy, on either side.** Relay's only middleware that can change what a tool *returned* is the execution intercept, which wraps the callback and therefore owns execution. pi never hands the callback over — it runs the tool in diff --git a/integrations/pi/index.ts b/integrations/pi/index.ts index 94dc0b4ce..ad7564e17 100644 --- a/integrations/pi/index.ts +++ b/integrations/pi/index.ts @@ -82,6 +82,7 @@ import { } from './src/user-bash.ts'; import type { AgentEndEvent, + BeforeProviderHeadersEvent, AgentSettledEvent, AgentStartEvent, ExtensionAPI, @@ -96,6 +97,7 @@ import type { ToolExecutionEndEvent, ToolExecutionStartEvent, TurnEndEvent, + PiModel, TurnStartEvent, UserBashEvent, UserBashEventResult, @@ -212,6 +214,24 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { /** Providers already pointed at the gateway, so the redirect is idempotent. */ const redirectedProviders = new Set(); let redirect: RedirectConfig | null = null; + /** + * Each provider's catalog as it was *before* we touched it. + * + * Snapshotted on first sight and never refreshed, because that is the whole + * point: once `registerProvider` has run, every model of that provider reports + * the gateway's URL, so re-reading the registry would compare the gateway + * against itself and call any provider safe. + */ + const pristineCatalog = new Map(); + + const siblingsOf = (ctx: ExtensionContext, provider: string): PiModel[] => { + const cached = pristineCatalog.get(provider); + if (cached) return cached; + const all = ctx.modelRegistry?.getAll?.() ?? []; + const models = all.filter((candidate) => candidate.provider === provider); + pristineCatalog.set(provider, models); + return models; + }; /** * Point the active model's provider at the gateway, when that is safe. @@ -225,7 +245,12 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { const applyRedirect = (ctx: ExtensionContext, source: string): void => { const active = ensureConfig(ctx); redirect ??= redirectConfigFromEnv(active.url); - const decision = decideRedirect(ctx.model, redirect, redirectedProviders); + const decision = decideRedirect( + ctx.model, + redirect, + redirectedProviders, + ctx.model ? siblingsOf(ctx, ctx.model.provider) : [], + ); if (decision.kind === 'redirect') { // Only baseUrl: pi rewrites the URL of every existing model for this // provider and keeps their API, headers and costs. @@ -240,6 +265,9 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { hook_event_name: 'model_redirect', source, outcome: decision.kind, + // The stable identifier, alongside the prose. Docs tell operators to interpret + // these codes, so a consumer must not have to pattern-match a human sentence. + ...(decision.kind === 'skip' ? { code: decision.code } : {}), reason: decision.reason, ...(decision.provider ? { provider: decision.provider } : {}), ...(decision.api ? { model_api: decision.api } : {}), @@ -262,6 +290,29 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { applyRedirect(ctx, 'session_start'); }); + /** + * Put the session id on redirected model requests. + * + * Hook posts carry `x-nemo-relay-session-id`; a redirected provider request is + * the extension's other stream and carried nothing, so with two pi sessions + * against one gateway an unkeyed model call is deliberately given an isolated + * root -- separating its LLM spans from the session and turn they belong to. + * + * Gated on `redirectedProviders`, because the hook is **global**: it fires for + * every provider request, including models we deliberately did not redirect. + * Sending an internal session id to a third-party provider is not acceptable + * just to simplify the handler. + * + * The id is read live rather than from the cached config, which is pinned for + * the runtime's life and would go stale across a session replacement. Headers + * are mutated in place; a returned value is ignored. + */ + pi.on('before_provider_headers', async (event: BeforeProviderHeadersEvent, ctx) => { + const provider = ctx.model?.provider; + if (!provider || !redirectedProviders.has(provider)) return; + event.headers['x-nemo-relay-session-id'] = safeSessionId(ctx); + }); + /** * Re-evaluate on every model switch. * @@ -501,7 +552,9 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { const endUserBash = ( ctx: ExtensionContext, callId: string, - status: 'ok' | 'error', + // `policy-allowed` rather than `ok`: pi reports no completion for inline shell, + // so the gate knows what it decided and never what happened. + status: 'policy-allowed' | 'error', content: string, ): void => { emit(ctx, { @@ -592,7 +645,11 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { return { result: refusalResult(reason) }; } - endUserBash(ctx, callId, 'ok', 'Allowed by policy; pi executed the command.'); + // Deliberately not `ok`: pi has not run the command yet, and has no completion + // hook to tell us how it went. It can still fail, and a later `user_bash` + // handler can replace execution entirely -- so claiming success here would put + // a false outcome in the trace. What we know is what we decided. + endUserBash(ctx, callId, 'policy-allowed', 'Allowed by policy; pi has not reported the outcome.'); // `undefined` is the only correct allow value: any object at all is a // result or an operations override, and either would stop pi running // the command as the user typed it. diff --git a/integrations/pi/package.json b/integrations/pi/package.json index 192fdd3c4..382fb68a5 100644 --- a/integrations/pi/package.json +++ b/integrations/pi/package.json @@ -10,6 +10,11 @@ "directory": "integrations/pi" }, "main": "./index.ts", + "pi": { + "extensions": [ + "./index.ts" + ] + }, "scripts": { "typecheck": "tsc -p tsconfig.json", "test": "node --test test/*.test.mjs" diff --git a/integrations/pi/src/pi-hook-types.ts b/integrations/pi/src/pi-hook-types.ts index 28831e3a9..7e61378d7 100644 --- a/integrations/pi/src/pi-hook-types.ts +++ b/integrations/pi/src/pi-hook-types.ts @@ -125,7 +125,15 @@ export type ToolCallEvent = { input: Record; }; -/** Returning `{block: true}` short-circuits the remaining `tool_call` handlers. */ +/** + * Returning `{block: true}` short-circuits the remaining `tool_call` handlers. + * + * ⚠️ Nothing else does. A truthy result without `block` is *retained* but does not + * stop iteration, so a handler that runs after this one still sees -- and can + * still mutate -- the same `input` object, with no re-validation before it + * executes. Loading first protects against being pre-empted; it does not make the + * verdict final. + */ export type ToolCallEventResult = { block?: boolean; reason?: string; @@ -177,6 +185,18 @@ export type PiModel = { baseUrl: string; }; +/** + * Fired before a provider request goes out, to let an extension add headers. + * + * `headers` is mutated in place; a returned object is ignored. The event carries + * no model or provider, so a handler that must scope itself reads `ctx.model` -- + * which is the *currently selected* model and can drift from the request's. + */ +export type BeforeProviderHeadersEvent = { + type: 'before_provider_headers'; + headers: Record; +}; + /** Fired when a model is selected, including the initial selection. */ export type ModelSelectEvent = { type: 'model_select'; @@ -193,6 +213,15 @@ export type ExtensionContext = { sessionManager: { getSessionId(): string }; /** The active model. Undefined before one is resolved. */ model?: PiModel; + /** + * The catalog, used to see a provider's *other* models before redirecting it. + * + * `registerProvider(name, {baseUrl})` rewrites every model of the provider, so + * a decision taken from the active model alone silently moves its siblings. + * Optional because a caller may not supply one; without it the check degrades + * to per-model, which is unsound for a provider that mixes API families. + */ + modelRegistry?: { getAll(): PiModel[] }; }; export type ExtensionHandler = ( @@ -216,6 +245,10 @@ export type ExtensionAPI = { on(event: 'tool_call', handler: ExtensionHandler): void; on(event: 'user_bash', handler: ExtensionHandler): void; on(event: 'model_select', handler: ExtensionHandler): void; + on( + event: 'before_provider_headers', + handler: ExtensionHandler, + ): void; /** * Register or override a model provider. diff --git a/integrations/pi/src/provider-redirect.ts b/integrations/pi/src/provider-redirect.ts index 45e8a3be8..57bdf2329 100644 --- a/integrations/pi/src/provider-redirect.ts +++ b/integrations/pi/src/provider-redirect.ts @@ -77,12 +77,28 @@ export type RedirectSkipCode = | 'already-redirected' | 'unserviceable-api' | 'unknown-upstream' - | 'upstream-mismatch'; + | 'upstream-mismatch' + | 'provider-mixed-endpoints'; export type RedirectDecision = | { kind: 'redirect'; provider: string; api: string; upstream: string; reason: string } | { kind: 'skip'; code: RedirectSkipCode; provider?: string; api?: string; reason: string }; +/** + * Whether one model could be pointed at the gateway without breaking it. + * + * The same two conditions the selected model must satisfy: the gateway serves its + * API family, and it already targets the endpoint the gateway forwards that family + * to. Applied to every sibling because the registration is provider-wide. + */ +function isSafeToRedirect(model: RedirectModel, config: RedirectConfig): boolean { + const family = SERVICEABLE_APIS[model.api]; + if (!family) return false; + const upstream = family === 'openai' ? config.openaiUpstream : config.anthropicUpstream; + if (!upstream) return false; + return normalizeBaseUrl(upstream) === normalizeBaseUrl(model.baseUrl); +} + /** Whether this outcome explains something a trace reader would otherwise have to guess. */ export function isNotable(decision: RedirectDecision): boolean { return decision.kind === 'redirect' || !['no-model', 'already-redirected'].includes(decision.code); @@ -121,6 +137,12 @@ export function decideRedirect( model: RedirectModel | undefined, config: RedirectConfig, redirected: ReadonlySet, + /** + * Every model of `model.provider`, as the catalog had them **before** any + * redirect. Optional only so an older caller still type-checks; omitting it + * restores the per-model check, which is unsound for a mixed provider. + */ + siblings: readonly RedirectModel[] = [], ): RedirectDecision { if (config.mode === 'off') { return { @@ -188,6 +210,28 @@ export function decideRedirect( }; } + // `registerProvider(name, {baseUrl})` rewrites the URL of EVERY model of that + // provider, so a decision made from the selected model alone is a decision made + // on behalf of its siblings. Several pi 0.84 providers mix API families at + // different paths -- Fireworks serves anthropic-messages at `/inference` and + // openai-completions at `/inference/v1`; opencode adds Google models the gateway + // cannot route at all -- so redirecting on the strength of one model points the + // others somewhere that has never heard of them. That is not "no spans", it is a + // broken session, mid-run, after the user changed only the model. + const unsafe = siblings.find((sibling) => !isSafeToRedirect(sibling, config)); + if (unsafe) { + return { + kind: 'skip', + code: 'provider-mixed-endpoints', + provider: model.provider, + api: model.api, + reason: + `redirecting ${model.provider} would also move its ${unsafe.api} models, and ` + + `${unsafe.id} targets ${unsafe.baseUrl}, which the gateway does not front; ` + + `point both upstreams at ${model.provider}, or accept no LLM spans for it`, + }; + } + if (normalizeBaseUrl(upstream) !== normalizeBaseUrl(model.baseUrl)) { return { kind: 'skip', diff --git a/integrations/pi/test/provider-redirect.test.mjs b/integrations/pi/test/provider-redirect.test.mjs index c226dda8c..34d530788 100644 --- a/integrations/pi/test/provider-redirect.test.mjs +++ b/integrations/pi/test/provider-redirect.test.mjs @@ -167,3 +167,66 @@ describe('base URL comparison', () => { ); }); }); + +// The failure this exists for, built from pi 0.84.0's real Fireworks catalog: +// 18 anthropic-messages models at `/inference` and 4 openai-completions models at +// `/inference/v1`, under one provider. `registerProvider(name, {baseUrl})` rewrites +// every model of a provider, so deciding from the selected model alone moves its +// siblings to an endpoint that has never heard of them. +describe('a provider whose models span API families', () => { + const ANTHROPIC = { + id: 'accounts/fireworks/models/deepseek-v4-flash', + api: 'anthropic-messages', + provider: 'fireworks', + baseUrl: 'https://api.fireworks.ai/inference', + }; + const OPENAI = { + id: 'accounts/fireworks/models/glm-5p2', + api: 'openai-completions', + provider: 'fireworks', + baseUrl: 'https://api.fireworks.ai/inference/v1', + }; + const catalog = [ANTHROPIC, OPENAI]; + + it('refuses to redirect when a sibling would be sent somewhere the gateway does not front', () => { + const config = { + gatewayUrl: 'http://127.0.0.1:4040', + mode: 'match', + anthropicUpstream: 'https://api.fireworks.ai/inference', + openaiUpstream: 'https://api.openai.com/v1', + }; + // Judged alone, the selected model looks perfectly safe. + assert.equal(decideRedirect(ANTHROPIC, config, new Set()).kind, 'redirect'); + + // Judged with its siblings, it is not: the openai-completions models would be + // pointed at api.openai.com carrying a Fireworks key and a Fireworks model id. + const decision = decideRedirect(ANTHROPIC, config, new Set(), catalog); + assert.equal(decision.kind, 'skip'); + assert.equal(decision.code, 'provider-mixed-endpoints'); + assert.match(decision.reason, /glm-5p2/, 'the reason must name the sibling that fails'); + assert.match(decision.reason, /openai-completions/); + }); + + it('redirects once both upstreams point at that provider', () => { + const config = { + gatewayUrl: 'http://127.0.0.1:4040', + mode: 'match', + anthropicUpstream: 'https://api.fireworks.ai/inference', + openaiUpstream: 'https://api.fireworks.ai/inference/v1', + }; + for (const model of catalog) { + const decision = decideRedirect(model, config, new Set(), catalog); + assert.equal(decision.kind, 'redirect', `${model.api} should redirect: ${decision.reason}`); + } + }); + + it('is notable, so the trace explains why there are no LLM spans', () => { + const config = { + gatewayUrl: 'http://127.0.0.1:4040', + mode: 'match', + anthropicUpstream: 'https://api.fireworks.ai/inference', + openaiUpstream: 'https://api.openai.com/v1', + }; + assert.equal(isNotable(decideRedirect(ANTHROPIC, config, new Set(), catalog)), true); + }); +}); diff --git a/integrations/pi/test/user-bash.test.mjs b/integrations/pi/test/user-bash.test.mjs index f2537ca28..e8fffb9ed 100644 --- a/integrations/pi/test/user-bash.test.mjs +++ b/integrations/pi/test/user-bash.test.mjs @@ -84,7 +84,9 @@ describe('inline shell gate', () => { await drain(fire); const [close] = named(gateway.posts, 'user_bash_end'); assert.ok(close, 'the gate span must close even when the command is allowed'); - assert.equal(close.status, 'ok'); + // Not `ok`: pi has not run it yet and never reports how it went, so the span + // records the decision rather than claiming an outcome it cannot know. + assert.equal(close.status, 'policy-allowed'); }); it('refuses a blocked command with a failed result carrying the reason verbatim', async () => { From de05c0ff3fd5764f51e12b7ff36d92e5c6b1a64c Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 23:07:11 -0700 Subject: [PATCH 19/41] fix(pi): identify this extension, and scope the session key by construction **The doctor checked for *a* pi extension, not *this* one.** Any non-empty `packages` array counted, and any directory entry counted -- so a user with somebody else's pi package got a Relay `Pass` for the load path, and an unrelated project-scoped install produced a Relay trust warning about a file with nothing to do with Relay. Both now resolve to the `nemo-relay-pi` manifest: a package directory whose `package.json` names it, or a file inside one. A local settings source is resolved relative to the settings file; an npm or git source is a name rather than a location, so those match on the specifier, which is the best signal short of fetching. `hook_status` shared the defect from the other side -- it read only the environment variable, so one `doctor` run could report "pi extension not located" beside a passing load path for the same machine. Both now answer from the same resolution, and a test pins that they agree. **The session key was scoped by a guess.** It rode on a `before_provider_headers` handler gated on `ctx.model?.provider`, but that hook carries no request identity: its event holds only headers, and its context is built fresh, so `ctx.model` is whatever is selected *now*. Both directions were wrong -- omit the key from a redirected call whose model was captured before a switch, or send an internal session id to a third-party provider we deliberately did not redirect. It moves to the `registerProvider` config, where the scope is structural: only providers actually pointed at the gateway carry it. That freezes the value at registration, so a session replacement (`/new`, `/resume`, `/fork` keep the runtime alive with a new id) would otherwise leave every provider stamping the old one -- the redirect set is now invalidated when the id moves, and re-registers. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- crates/cli/src/agents/pi/doctor.rs | 145 +++++++++++++----- crates/cli/src/diagnostics/mod.rs | 17 +- .../tests/coverage/agents/pi_doctor_tests.rs | 90 +++++++++-- integrations/pi/index.ts | 55 +++---- integrations/pi/src/pi-hook-types.ts | 21 +-- integrations/pi/test/lifecycle.test.mjs | 89 +++++++++++ 6 files changed, 308 insertions(+), 109 deletions(-) diff --git a/crates/cli/src/agents/pi/doctor.rs b/crates/cli/src/agents/pi/doctor.rs index 012fdb105..6bf797cc2 100644 --- a/crates/cli/src/agents/pi/doctor.rs +++ b/crates/cli/src/agents/pi/doctor.rs @@ -34,6 +34,10 @@ const PI_CONFIG_DIR: &str = ".pi"; /// Where pi records installed packages, in both scopes. const PI_SETTINGS_FILE: &str = "settings.json"; +/// This extension's package name -- how it is told apart from anyone else's. +/// Must match `integrations/pi/package.json`. +const RELAY_PACKAGE_NAME: &str = "nemo-relay-pi"; + /// Gateway URL the extension falls back to when nothing else resolves one. /// Kept in step with `configFromEnv` in `integrations/pi/src/gateway-client.ts`. const DEFAULT_GATEWAY_URL: &str = "http://127.0.0.1:4040"; @@ -51,6 +55,17 @@ pub(crate) enum ExtensionScope { Project, } +impl ExtensionScope { + /// How this route behaves, in the words the check reports. + pub(crate) fn describe(self) -> &'static str { + match self { + Self::Explicit => "passed with `-e`, which loads first and is never trust-gated", + Self::User => "user scope, which is never trust-gated", + Self::Project => "project scope, which pi loads only for a trusted project", + } + } +} + /// A place a pi extension was found, and how pi would reach it. #[derive(Debug, Clone)] pub(crate) struct ExtensionSite { @@ -59,23 +74,33 @@ pub(crate) struct ExtensionSite { } /// Human-readable hook status for `nemo-relay doctor`. +/// +/// Shares its answer with the load-path check, deliberately. While this read only +/// the environment variable and that scanned directories, one `doctor` run could +/// report "pi extension not located" *and* a passing load path for the same +/// machine, in the same output. pub(crate) fn hook_status() -> Result { - match extension_location() { - Some(path) => Ok(format!( - "pi extension resolved at {} (hooks are emitted by the extension, not by pi itself)", - path.display() + match relay_extension_sites(¤t_dir()).first() { + Some(site) => Ok(format!( + "NeMo Relay pi extension resolved at {} ({}); hooks are emitted by the extension, \ + not by pi itself", + site.path.display(), + site.scope.describe() )), None => Ok(format!( - "pi extension not located; set {PI_EXTENSION_PATH_ENV}, or install the extension with \ - `pi install ` or into an auto-discovered directory \ - (`~/.pi/agent/extensions/`, `.pi/extensions/`)" + "NeMo Relay pi extension not located; set {PI_EXTENSION_PATH_ENV}, run \ + `pi install `, or copy it into `~/.pi/agent/extensions/`" )), } } -/// Whether the extension entry point can be found. +/// Whether *this* extension -- not merely some pi extension -- can be found. pub(crate) fn extension_configured() -> bool { - extension_location().is_some() + !relay_extension_sites(¤t_dir()).is_empty() +} + +fn current_dir() -> PathBuf { + std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")) } fn extension_location() -> Option { @@ -125,7 +150,7 @@ pub(crate) fn gateway_url(bind: Option) -> String { /// reported "no pi extension found" to a user who had just run the install /// command the docs recommend, and could not see a trust-gated `--local` entry /// at all, which is the case this whole module exists to catch. -pub(crate) fn extension_sites(cwd: &Path) -> Vec { +pub(crate) fn relay_extension_sites(cwd: &Path) -> Vec { let mut sites = Vec::new(); if let Some(path) = extension_location() { sites.push(ExtensionSite { @@ -134,62 +159,106 @@ pub(crate) fn extension_sites(cwd: &Path) -> Vec { }); } if let Some(dir) = user_extensions_dir() - && directory_has_entries(&dir) + && let Some(path) = relay_entry_in_directory(&dir) { sites.push(ExtensionSite { - path: dir, + path, scope: ExtensionScope::User, }); } - if let Some(path) = user_settings_path() - && settings_declare_packages(&path) + if let Some(settings) = user_settings_path() + && let Some(path) = relay_package_in_settings(&settings) { sites.push(ExtensionSite { path, scope: ExtensionScope::User, }); } - let project_dir = cwd.join(PI_CONFIG_DIR).join("extensions"); - if directory_has_entries(&project_dir) { + if let Some(path) = relay_entry_in_directory(&cwd.join(PI_CONFIG_DIR).join("extensions")) { sites.push(ExtensionSite { - path: project_dir, + path, scope: ExtensionScope::Project, }); } - let project_settings = cwd.join(PI_CONFIG_DIR).join(PI_SETTINGS_FILE); - if settings_declare_packages(&project_settings) { + if let Some(path) = relay_package_in_settings(&cwd.join(PI_CONFIG_DIR).join(PI_SETTINGS_FILE)) { sites.push(ExtensionSite { - path: project_settings, + path, scope: ExtensionScope::Project, }); } sites } -/// `/settings.json`, where `pi install` records a user-scope package. -fn user_settings_path() -> Option { - Some(pi_agent_dir()?.join(PI_SETTINGS_FILE)) +/// The NeMo Relay extension inside a pi auto-discovery directory, if it is there. +/// +/// Matched on the package name, not on "the directory is non-empty". A user with +/// somebody else's pi extension installed was otherwise told their *Relay* +/// extension was fine -- and, worse, a project-scoped install of an unrelated +/// package raised a Relay trust warning about a file that has nothing to do with +/// Relay. +fn relay_entry_in_directory(dir: &Path) -> Option { + std::fs::read_dir(dir) + .ok()? + .flatten() + .map(|entry| entry.path()) + .find(|path| is_relay_extension(path)) } -/// Whether a pi settings file declares at least one installed package. -/// -/// Deliberately tolerant: an unreadable or malformed settings file is reported as -/// "no packages" rather than as an error. This check exists to find something the -/// user installed, and a parse failure here is pi's problem to report, not a -/// reason for `doctor` to fail. -fn settings_declare_packages(path: &Path) -> bool { - std::fs::read_to_string(path) +/// Whether a path is this extension: a package directory whose manifest names it, +/// or a file sitting inside one. +fn is_relay_extension(path: &Path) -> bool { + if manifest_names_relay(&path.join("package.json")) { + return true; + } + path.parent() + .is_some_and(|parent| manifest_names_relay(&parent.join("package.json"))) +} + +fn manifest_names_relay(manifest: &Path) -> bool { + std::fs::read_to_string(manifest) .ok() .and_then(|raw| serde_json::from_str::(&raw).ok()) - .and_then(|settings| { - settings - .get("packages") - .and_then(serde_json::Value::as_array) - .map(|packages| !packages.is_empty()) + .and_then(|value| { + value + .get("name") + .and_then(serde_json::Value::as_str) + .map(|name| name == RELAY_PACKAGE_NAME) }) .unwrap_or(false) } +/// The NeMo Relay package among the sources `pi install` recorded, if any. +/// +/// Each entry is a source string, and a local one is a path relative to the +/// settings file's own directory. Only a local source can be resolved from here +/// -- an npm or git source is a name, not a location -- so those fall back to +/// matching the package name inside the specifier, which is the best signal +/// available without fetching anything. +fn relay_package_in_settings(settings: &Path) -> Option { + let base = settings.parent()?; + let raw = std::fs::read_to_string(settings).ok()?; + let value: serde_json::Value = serde_json::from_str(&raw).ok()?; + value + .get("packages")? + .as_array()? + .iter() + .filter_map(serde_json::Value::as_str) + .find_map(|source| { + let resolved = base.join(source); + if is_relay_extension(&resolved) { + return Some(resolved); + } + source + .contains(RELAY_PACKAGE_NAME) + .then(|| PathBuf::from(source)) + }) +} + +/// `/settings.json`, where `pi install` records a user-scope package. +fn user_settings_path() -> Option { + Some(pi_agent_dir()?.join(PI_SETTINGS_FILE)) +} + /// `~/.pi/agent`, honoring pi's own directory override. fn pi_agent_dir() -> Option { match std::env::var_os(PI_AGENT_DIR_ENV) { @@ -208,10 +277,6 @@ fn user_extensions_dir() -> Option { Some(pi_agent_dir()?.join("extensions")) } -fn directory_has_entries(path: &Path) -> bool { - std::fs::read_dir(path).is_ok_and(|mut entries| entries.next().is_some()) -} - #[cfg(test)] #[path = "../../../tests/coverage/agents/pi_doctor_tests.rs"] mod tests; diff --git a/crates/cli/src/diagnostics/mod.rs b/crates/cli/src/diagnostics/mod.rs index 293c72ec2..35783d499 100644 --- a/crates/cli/src/diagnostics/mod.rs +++ b/crates/cli/src/diagnostics/mod.rs @@ -520,7 +520,7 @@ async fn agent_preflight_checks( /// place a user finds out before wondering why NeMo Relay "does nothing". fn pi_extension_trust_check(cwd: &Path) -> Check { const NAME: &str = "pi extension load path"; - let sites = crate::agents::pi::doctor::extension_sites(cwd); + let sites = crate::agents::pi::doctor::relay_extension_sites(cwd); let project_sites: Vec<&crate::agents::pi::doctor::ExtensionSite> = sites .iter() .filter(|site| site.scope == crate::agents::pi::doctor::ExtensionScope::Project) @@ -545,23 +545,14 @@ fn pi_extension_trust_check(cwd: &Path) -> Check { Some(site) => Check { name: NAME, status: Status::Pass, - details: format!( - "{} ({})", - site.path.display(), - match site.scope { - crate::agents::pi::doctor::ExtensionScope::Explicit => - "passed with `-e`, which loads first and is never trust-gated", - crate::agents::pi::doctor::ExtensionScope::User => - "user scope, which is never trust-gated", - crate::agents::pi::doctor::ExtensionScope::Project => "project scope", - } - ), + details: format!("{} ({})", site.path.display(), site.scope.describe()), }, None => Check { name: NAME, status: Status::Info, details: format!( - "no pi extension found; set {} or install one into `~/.pi/agent/extensions/`", + "the NeMo Relay pi extension was not found; set {} or install it with \ + `pi install `", crate::agents::pi::launch::PI_EXTENSION_PATH_ENV ), }, diff --git a/crates/cli/tests/coverage/agents/pi_doctor_tests.rs b/crates/cli/tests/coverage/agents/pi_doctor_tests.rs index 5ef8e8b7d..0160fd9a1 100644 --- a/crates/cli/tests/coverage/agents/pi_doctor_tests.rs +++ b/crates/cli/tests/coverage/agents/pi_doctor_tests.rs @@ -8,6 +8,24 @@ use crate::test_support::EnvScope; /// Isolate the three environment variables that steer extension discovery, so a /// developer's own pi install cannot make these pass or fail. +/// A directory pi would see as this extension: a package manifest naming it. +fn write_relay_package(dir: &std::path::Path) { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write(dir.join("package.json"), r#"{"name": "nemo-relay-pi"}"#).unwrap(); + std::fs::write(dir.join("index.ts"), "export default 1").unwrap(); +} + +/// Somebody else's pi extension, installed the same way. +fn write_other_package(dir: &std::path::Path) { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write( + dir.join("package.json"), + r#"{"name": "someone-elses-pi-thing"}"#, + ) + .unwrap(); + std::fs::write(dir.join("index.ts"), "export default 1").unwrap(); +} + fn scoped(extension: Option<&OsStr>, agent_dir: Option<&OsStr>) -> EnvScope { EnvScope::set(&[ (PI_EXTENSION_PATH_ENV, extension), @@ -20,12 +38,12 @@ fn a_project_scoped_extension_is_reported_because_pi_will_not_say_so() { let temp = tempfile::tempdir().unwrap(); let project_extensions = temp.path().join(".pi").join("extensions"); std::fs::create_dir_all(&project_extensions).unwrap(); - std::fs::write(project_extensions.join("nemo-relay.ts"), "export default 1").unwrap(); + write_relay_package(&project_extensions.join("nemo-relay")); let empty_home = temp.path().join("home"); std::fs::create_dir_all(&empty_home).unwrap(); let _env = scoped(None, Some(empty_home.as_os_str())); - let sites = extension_sites(temp.path()); + let sites = relay_extension_sites(temp.path()); // This is the whole point of the check: pi drops this extension with a bare // conditional in every non-interactive mode, never reports it, and the @@ -50,7 +68,7 @@ fn an_empty_project_directory_is_not_reported() { // A `.pi/extensions` directory that pi created and nothing was ever put in // is not a finding; warning about it would train users to ignore the check. assert!( - extension_sites(temp.path()).is_empty(), + relay_extension_sites(temp.path()).is_empty(), "an empty project extensions directory must not be reported" ); } @@ -64,7 +82,7 @@ fn the_explicit_path_is_reported_as_ungated() { std::fs::create_dir_all(&empty_home).unwrap(); let _env = scoped(Some(entry.as_os_str()), Some(empty_home.as_os_str())); - let sites = extension_sites(temp.path()); + let sites = relay_extension_sites(temp.path()); // `-e` loads first in pi's precedence order and survives `--no-extensions`, // so an extension reached this way is never subject to project trust -- @@ -80,10 +98,10 @@ fn a_user_scope_install_is_reported_as_ungated() { let agent_dir = temp.path().join("agent"); let user_extensions = agent_dir.join("extensions"); std::fs::create_dir_all(&user_extensions).unwrap(); - std::fs::write(user_extensions.join("nemo-relay.ts"), "export default 1").unwrap(); + write_relay_package(&user_extensions.join("nemo-relay")); let _env = scoped(None, Some(agent_dir.as_os_str())); - let sites = extension_sites(temp.path()); + let sites = relay_extension_sites(temp.path()); assert_eq!(sites.len(), 1, "{sites:?}"); assert_eq!(sites[0].scope, ExtensionScope::User); @@ -100,7 +118,7 @@ fn an_explicit_path_that_does_not_exist_is_not_reported() { // A stale environment variable is worse than none: it would report an // ungated load path for a file pi cannot read. - assert!(extension_sites(temp.path()).is_empty()); + assert!(relay_extension_sites(temp.path()).is_empty()); assert!(!extension_configured()); } @@ -113,14 +131,15 @@ fn a_pi_install_at_user_scope_is_found_in_settings_not_in_a_directory() { let temp = tempfile::tempdir().unwrap(); let agent_dir = temp.path().join("agent"); std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("checkout")); std::fs::write( agent_dir.join("settings.json"), - r#"{"packages": ["../../../NeMo-Relay/integrations/pi"]}"#, + r#"{"packages": ["../checkout"]}"#, ) .unwrap(); let _env = scoped(None, Some(agent_dir.as_os_str())); - let sites = extension_sites(temp.path()); + let sites = relay_extension_sites(temp.path()); assert_eq!(sites.len(), 1, "{sites:?}"); assert_eq!(sites[0].scope, ExtensionScope::User); @@ -133,16 +152,17 @@ fn a_pi_install_at_user_scope_is_found_in_settings_not_in_a_directory() { fn a_local_pi_install_is_reported_as_project_scoped() { let temp = tempfile::tempdir().unwrap(); std::fs::create_dir_all(temp.path().join(".pi")).unwrap(); + write_relay_package(&temp.path().join("checkout")); std::fs::write( temp.path().join(".pi").join("settings.json"), - r#"{"packages": ["../extensions/nemo-relay"]}"#, + r#"{"packages": ["../checkout"]}"#, ) .unwrap(); let empty_home = temp.path().join("home"); std::fs::create_dir_all(&empty_home).unwrap(); let _env = scoped(None, Some(empty_home.as_os_str())); - let sites = extension_sites(temp.path()); + let sites = relay_extension_sites(temp.path()); assert!( sites @@ -169,7 +189,7 @@ fn settings_without_packages_are_not_a_finding() { ] { std::fs::write(agent_dir.join("settings.json"), body).unwrap(); assert!( - extension_sites(temp.path()).is_empty(), + relay_extension_sites(temp.path()).is_empty(), "settings body {body} must not be reported as an install" ); } @@ -227,3 +247,49 @@ fn the_environment_variable_wins_over_a_configured_bind() { "http://elsewhere:1234" ); } + +// The check is about *this* extension, not about pi having extensions. Before it +// matched on the package name, an unrelated install produced a Relay Pass -- and a +// project-scoped one produced a Relay trust warning about a file that has nothing +// to do with Relay. +#[test] +fn somebody_elses_pi_extension_is_not_reported_as_ours() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + write_other_package(&agent_dir.join("extensions").join("other")); + write_other_package(&temp.path().join(".pi").join("extensions").join("other")); + std::fs::create_dir_all(temp.path().join(".pi")).unwrap(); + write_other_package(&temp.path().join("elsewhere")); + std::fs::write( + temp.path().join(".pi").join("settings.json"), + r#"{"packages": ["../elsewhere"]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + assert!( + relay_extension_sites(temp.path()).is_empty(), + "an unrelated pi package must not be reported as the Relay extension" + ); + assert!(!extension_configured()); +} + +// The headline status and the load-path check must agree: they now answer from the +// same resolution, so one `doctor` run cannot say "not located" beside a Pass. +#[test] +fn the_headline_status_agrees_with_the_load_path_check() { + let temp = tempfile::tempdir().unwrap(); + let entry = temp.path().join("index.ts"); + std::fs::write(&entry, "export default 1").unwrap(); + let empty_home = temp.path().join("home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let _found = scoped(Some(entry.as_os_str()), Some(empty_home.as_os_str())); + assert!(extension_configured()); + assert!(hook_status().unwrap().contains("resolved at")); + drop(_found); + + let _missing = scoped(None, Some(empty_home.as_os_str())); + assert!(!extension_configured()); + assert!(hook_status().unwrap().contains("not located")); +} diff --git a/integrations/pi/index.ts b/integrations/pi/index.ts index ad7564e17..3e02b1f0e 100644 --- a/integrations/pi/index.ts +++ b/integrations/pi/index.ts @@ -82,7 +82,6 @@ import { } from './src/user-bash.ts'; import type { AgentEndEvent, - BeforeProviderHeadersEvent, AgentSettledEvent, AgentStartEvent, ExtensionAPI, @@ -213,6 +212,8 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { /** Providers already pointed at the gateway, so the redirect is idempotent. */ const redirectedProviders = new Set(); + /** Session id the current registrations carry, so a replacement can invalidate them. */ + let registeredSessionKey: string | null = null; let redirect: RedirectConfig | null = null; /** * Each provider's catalog as it was *before* we touched it. @@ -245,6 +246,16 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { const applyRedirect = (ctx: ExtensionContext, source: string): void => { const active = ensureConfig(ctx); redirect ??= redirectConfigFromEnv(active.url); + + // The join key is baked into the registration, so it cannot follow a session + // replacement on its own: `/new`, `/resume` and `/fork` keep this runtime + // alive and give it a new session id, leaving every redirected provider + // stamping the old one onto its requests. Re-register when it moves. + const sessionKey = safeSessionId(ctx); + if (registeredSessionKey !== null && registeredSessionKey !== sessionKey) { + redirectedProviders.clear(); + } + registeredSessionKey = sessionKey; const decision = decideRedirect( ctx.model, redirect, @@ -252,9 +263,22 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { ctx.model ? siblingsOf(ctx, ctx.model.provider) : [], ); if (decision.kind === 'redirect') { - // Only baseUrl: pi rewrites the URL of every existing model for this - // provider and keeps their API, headers and costs. - pi.registerProvider(decision.provider, { baseUrl: redirect.gatewayUrl }); + // `baseUrl` rewrites the URL of every existing model for this provider and + // keeps their API and costs. `headers` is the session join key. + // + // It goes here rather than in a `before_provider_headers` handler because + // that hook is **global and carries no request identity**: its event has + // only the headers, and its context is freshly built, so `ctx.model` is + // whatever is selected *now*, not what this request is for. Scoping on it + // gets both directions wrong -- omitting the key from a redirected call + // whose model was captured before a switch, and leaking an internal session + // id to a third-party provider we deliberately did not redirect. Attaching + // it to the registration makes the scope structural: only providers we + // actually pointed at the gateway ever send it. + pi.registerProvider(decision.provider, { + baseUrl: redirect.gatewayUrl, + headers: { 'x-nemo-relay-session-id': sessionKey }, + }); redirectedProviders.add(decision.provider); } // A transient skip -- no model resolved yet, or a provider already pointed @@ -290,29 +314,6 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { applyRedirect(ctx, 'session_start'); }); - /** - * Put the session id on redirected model requests. - * - * Hook posts carry `x-nemo-relay-session-id`; a redirected provider request is - * the extension's other stream and carried nothing, so with two pi sessions - * against one gateway an unkeyed model call is deliberately given an isolated - * root -- separating its LLM spans from the session and turn they belong to. - * - * Gated on `redirectedProviders`, because the hook is **global**: it fires for - * every provider request, including models we deliberately did not redirect. - * Sending an internal session id to a third-party provider is not acceptable - * just to simplify the handler. - * - * The id is read live rather than from the cached config, which is pinned for - * the runtime's life and would go stale across a session replacement. Headers - * are mutated in place; a returned value is ignored. - */ - pi.on('before_provider_headers', async (event: BeforeProviderHeadersEvent, ctx) => { - const provider = ctx.model?.provider; - if (!provider || !redirectedProviders.has(provider)) return; - event.headers['x-nemo-relay-session-id'] = safeSessionId(ctx); - }); - /** * Re-evaluate on every model switch. * diff --git a/integrations/pi/src/pi-hook-types.ts b/integrations/pi/src/pi-hook-types.ts index 7e61378d7..06f484d5a 100644 --- a/integrations/pi/src/pi-hook-types.ts +++ b/integrations/pi/src/pi-hook-types.ts @@ -185,18 +185,6 @@ export type PiModel = { baseUrl: string; }; -/** - * Fired before a provider request goes out, to let an extension add headers. - * - * `headers` is mutated in place; a returned object is ignored. The event carries - * no model or provider, so a handler that must scope itself reads `ctx.model` -- - * which is the *currently selected* model and can drift from the request's. - */ -export type BeforeProviderHeadersEvent = { - type: 'before_provider_headers'; - headers: Record; -}; - /** Fired when a model is selected, including the initial selection. */ export type ModelSelectEvent = { type: 'model_select'; @@ -245,10 +233,6 @@ export type ExtensionAPI = { on(event: 'tool_call', handler: ExtensionHandler): void; on(event: 'user_bash', handler: ExtensionHandler): void; on(event: 'model_select', handler: ExtensionHandler): void; - on( - event: 'before_provider_headers', - handler: ExtensionHandler, - ): void; /** * Register or override a model provider. @@ -259,5 +243,8 @@ export type ExtensionAPI = { * queued and applied once the runner binds its context, so calling it from a * factory is safe. */ - registerProvider(name: string, config: { baseUrl?: string }): void; + registerProvider( + name: string, + config: { baseUrl?: string; headers?: Record }, + ): void; }; diff --git a/integrations/pi/test/lifecycle.test.mjs b/integrations/pi/test/lifecycle.test.mjs index 4878ee208..a65357a6a 100644 --- a/integrations/pi/test/lifecycle.test.mjs +++ b/integrations/pi/test/lifecycle.test.mjs @@ -596,3 +596,92 @@ describe('what an interrupted session loses', () => { ); }); }); + +describe('the session join key on redirected providers', () => { + let ctx; + + before(async () => { + ctx = stubGateway(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = await listen(ctx.server); + process.env.NEMO_RELAY_PI_OPENAI_UPSTREAM = 'https://api.openai.com/v1'; + }); + + after(() => { + ctx.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + delete process.env.NEMO_RELAY_PI_OPENAI_UPSTREAM; + }); + + beforeEach(() => { + ctx.posts.length = 0; + }); + + const model = { + id: 'gpt-test', + api: 'openai-completions', + provider: 'openai', + baseUrl: 'https://api.openai.com/v1', + }; + + /** Drive the extension while recording what it registers, and with a settable session id. */ + function loadRecording(sessionId) { + const registrations = []; + const handlers = new Map(); + let current = sessionId; + extension({ + on(name, handler) { + if (!handlers.has(name)) handlers.set(name, []); + handlers.get(name).push(handler); + }, + registerProvider: (name, config) => registrations.push({ name, config }), + }); + const context = { + cwd: '/work', + mode: 'interactive', + hasUI: true, + sessionManager: { getSessionId: () => current }, + model, + modelRegistry: { getAll: () => [model] }, + }; + const fire = async (name, event = {}) => { + for (const handler of handlers.get(name) ?? []) { + await handler({ type: name, ...event }, context); + } + }; + return { fire, registrations, setSession: (id) => (current = id) }; + } + + // The header rides on the registration, not on a per-request hook: pi's + // `before_provider_headers` carries no request identity, and its context reports + // the *currently selected* model -- so scoping on it would both omit the key from + // a redirected call whose model was captured before a switch, and leak an internal + // session id to a provider we deliberately did not redirect. + it('rides on the provider registration, so only redirected providers send it', async () => { + const { fire, registrations } = loadRecording('sess-one'); + await fire('session_start', { reason: 'startup' }); + + assert.equal(registrations.length, 1, 'the matching provider should be redirected'); + assert.equal(registrations[0].name, 'openai'); + assert.equal(registrations[0].config.headers['x-nemo-relay-session-id'], 'sess-one'); + }); + + // The key is baked in at registration, so it cannot follow a replacement on its + // own -- `/new`, `/resume` and `/fork` keep this runtime alive with a new id. + it('is refreshed when the session is replaced under the same runtime', async () => { + const { fire, registrations, setSession } = loadRecording('sess-one'); + await fire('session_start', { reason: 'startup' }); + + setSession('sess-two'); + await fire('session_start', { reason: 'resume' }); + + assert.equal(registrations.length, 2, 'a replacement must re-register, not reuse'); + assert.equal(registrations[1].config.headers['x-nemo-relay-session-id'], 'sess-two'); + }); + + it('does not re-register when the session id has not moved', async () => { + const { fire, registrations } = loadRecording('sess-one'); + await fire('session_start', { reason: 'startup' }); + await fire('model_select', { model }); + assert.equal(registrations.length, 1, 'the redirect must stay idempotent'); + }); +}); From 9e94e4a036b0da172b095db8cb012d8c62986913 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Tue, 18 Aug 2026 23:51:23 -0700 Subject: [PATCH 20/41] docs(pi): the tool schema is reachable; the transform design is a choice The argument-transform notes said pi "exposes `tools` only on the `Extension` interface, which is an extension's own registered tools", so no built-in's schema could be read and neither local validation nor forwarding was possible. That is false: `pi.getAllTools()` returns every configured tool -- built-ins included -- with its TypeBox `parameters` schema (`extensions/types.ts:1334`, implemented at `core/agent-session.ts:908`, and pi's own docs show it returning `read` with `sourceInfo.source: "builtin"`). The shape-preserving check stands, but on its real footing. The schema is per-session mutable state -- `setActiveTools` and `registerTool` change the tool set mid-session -- so a schema read once can go stale, and forwarding it would make the gateway carry pi's tool vocabulary for a check that does not need it. Those are the reasons; "we cannot see it" was not one. "`pattern`, `enum` and range constraints are not checked **and cannot be**" is corrected to the same effect: they are not checked by choice. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- docs/nemo-relay-cli/pi.mdx | 6 ++++-- integrations/pi/README.md | 16 ++++++++++------ integrations/pi/src/argument-transform.ts | 20 ++++++++++++-------- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index 568b206a8..226fb7fb4 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -192,8 +192,10 @@ extension applies it to pi's `event.input` in place. The rewrite is **constrained, not validated**. pi validates tool arguments -before the hook and never re-validates, and the extension cannot read a built-in -tool's schema, so the extension enforces a shape invariant instead: a transform +before the hook and never re-validates. The extension could read the tool's +schema — `pi.getAllTools()` exposes it for built-ins too — but deliberately does +not: pi's tool set is per-session mutable, so a schema read once can go stale +mid-session. It enforces a shape invariant instead: a transform may rewrite the values of existing keys, preserving each value's JSON type. It may not add or remove keys, change a type, or change an array's length. diff --git a/integrations/pi/README.md b/integrations/pi/README.md index 4106ca223..004f416f0 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -144,10 +144,14 @@ later handlers, so replacing the reference would be discarded. ⚠️ **The rewrite is constrained, not validated.** pi validates arguments *before* the `tool_call` hook and never re-validates — its own types say "no re-validation is performed after mutation" — so a rewrite that violates the -tool's schema would execute. The extension cannot check it against the schema -either: pi exposes `tools` only on the `Extension` interface, which is an -extension's *own* registered tools, so there is no way to read the schema of a -built-in like `read`. +tool's schema would execute. + +The extension *could* check it: `pi.getAllTools()` returns every configured +tool — built-ins included — with its TypeBox `parameters` schema. That is +deliberately not used. pi's tool set is per-session mutable (`setActiveTools`, +`registerTool`), so a schema read once can go stale mid-session, and forwarding +it would make the gateway carry pi's tool vocabulary for a check that does not +need it. So the extension enforces a **shape** invariant instead: a transform may rewrite the values of existing keys, preserving each value's JSON type, recursively. @@ -156,8 +160,8 @@ refused. An argument object that satisfied the schema before therefore still has the required keys of the required types afterwards. **This is structural, not schema validation.** `pattern`, `enum`, `minimum` and -`format` are not checked and cannot be. A transform that rewrites a string to -one the schema would reject still executes. +`format` are not checked — by choice, not because the schema is out of reach. A +transform that rewrites a string to one the schema would reject still executes. A refused transform **blocks the call**, with a reason that says the policy could not be applied rather than that the request was refused. Running the original diff --git a/integrations/pi/src/argument-transform.ts b/integrations/pi/src/argument-transform.ts index 9e52de4fc..edc334ec3 100644 --- a/integrations/pi/src/argument-transform.ts +++ b/integrations/pi/src/argument-transform.ts @@ -15,21 +15,25 @@ * `tool_call` receives, and pi's own types say "later `tool_call` handlers * see earlier mutations. No re-validation is performed after mutation." So * arguments that violate the tool's schema will execute. - * 2. *The extension cannot see the schema.* pi exposes `tools` only on the - * `Extension` interface, which is an extension's own registered tools -- - * there is no accessor for the schema of a built-in like `read` or `bash`. - * Neither we nor the gateway can check the rewrite against it. + * 2. *The schema is reachable, and deliberately not used.* `pi.getAllTools()` + * returns every configured tool -- built-ins included -- with its TypeBox + * `parameters` schema (`extensions/types.ts:1334`, impl + * `core/agent-session.ts:908`). So validating locally, or forwarding the + * schema to the gateway, are both possible. They are not done because the + * tool set is per-session mutable (`setActiveTools`, `registerTool`), so a + * forwarded schema can go stale mid-session, and carrying pi's tool + * vocabulary into the gateway buys precision this transform does not need. * - * So the transform is constrained instead of validated: it may **rewrite the + * So the transform is constrained rather than validated: it may **rewrite the * values of existing keys, preserving each value's JSON type**, recursively. * Adding a key, removing a key, changing a type, or changing an array's length * is refused. An object that satisfied the schema before therefore still has * the required keys, of the required types, afterwards. * * ⚠️ **This is a structural guarantee, not schema validation.** Value-level - * constraints -- `pattern`, `enum`, `minimum`, `format` -- are not checked and - * cannot be. A transform that rewrites a string to one the schema would reject - * still executes. + * constraints -- `pattern`, `enum`, `minimum`, `format` -- are not checked. That + * is a choice, not a limit: see point 2. A transform that rewrites a string to + * one the schema would reject still executes. * * A refused transform **blocks the call**. Running the original arguments would * silently discard a policy decision, which is the failure the transform From acd42907efe8346beed253f3ef1425094b29a795 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 09:32:08 -0700 Subject: [PATCH 21/41] fix(pi): launch from the same resolution doctor reports, and check it is ours Two holes in extension resolution, opposite sides of the same split. `extension_location` accepted any path that existed. Every other discovery route matches on the `nemo-relay-pi` manifest name; the explicit one did not, so a stale or mistyped `NEMO_RELAY_PI_EXTENSION` naming somebody else's extension made `doctor` report a Pass *and* made the launcher hand that path to `-e`. A green check described a session with no Relay code in it. `extension_path` in the launcher then read the same variable a second time, and only that variable. Both install routes the README recommends -- `pi install ` and a file drop into `~/.pi/agent/extensions/` -- set no variable and no document tells a user to set one, so `doctor` resolved the install and reported the setup as ready while `nemo-relay run --agent pi` refused to start. The launcher now goes through `launchable_extension_path`, which is `relay_extension_sites` minus two things it must not promote: - **Project scope.** `-e` is never trust-gated, so promoting one would run repository code pi itself declined to trust -- undoing the gate the preflight exists to warn about. The launch error says so. - **A source that is not a path.** `pi install` can record an npm or git specifier, and pi resolves an `-e` argument as a package *source*, so handing one back could make a launch fetch from the network. Passing `-e` for something pi would have discovered anyway is safe: pi canonicalizes and de-duplicates the merged command-line and discovered sets before loading, and both routes resolve a package directory through the same `pi.extensions` manifest, so the extension registers its hooks once. Verified against pi v0.84.0 rather than assumed: `-e` accepts a package directory, and `mergePaths` is what makes the duplicate harmless. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- crates/cli/src/agents/pi/doctor.rs | 36 +++++- crates/cli/src/agents/pi/launch.rs | 24 ++-- .../tests/coverage/agents/launcher_tests.rs | 114 ++++++++++++++++-- .../tests/coverage/agents/pi_doctor_tests.rs | 84 ++++++++++++- docs/nemo-relay-cli/pi.mdx | 13 +- integrations/pi/README.md | 7 ++ 6 files changed, 252 insertions(+), 26 deletions(-) diff --git a/crates/cli/src/agents/pi/doctor.rs b/crates/cli/src/agents/pi/doctor.rs index 6bf797cc2..dd909fc5e 100644 --- a/crates/cli/src/agents/pi/doctor.rs +++ b/crates/cli/src/agents/pi/doctor.rs @@ -26,7 +26,7 @@ use super::launch::{PI_EXTENSION_PATH_ENV, PI_GATEWAY_URL_ENV}; /// /// Mirrors `getAgentDir()` (pi `config.ts:515-522`), including the environment /// override, so the preflight looks where pi will actually look. -const PI_AGENT_DIR_ENV: &str = "PI_CODING_AGENT_DIR"; +pub(crate) const PI_AGENT_DIR_ENV: &str = "PI_CODING_AGENT_DIR"; /// pi's configuration directory name, from its `piConfig.configDir`. const PI_CONFIG_DIR: &str = ".pi"; @@ -103,10 +103,17 @@ fn current_dir() -> PathBuf { std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")) } +/// The explicitly configured extension, but only when it is *this* extension. +/// +/// The name check is not redundant with the directory scans below. A stale or +/// mistyped variable that happens to name an existing path -- somebody else's +/// extension, or a checkout that no longer holds ours -- otherwise reported a +/// Pass here *and* made the launcher hand that path to `-e`, so pi loaded code +/// that emits no hooks while every Relay check said the setup was ready. fn extension_location() -> Option { std::env::var_os(PI_EXTENSION_PATH_ENV) .map(PathBuf::from) - .filter(|path| path.exists()) + .filter(|path| path.exists() && is_relay_extension(path)) } /// The gateway URL the extension will post to. @@ -189,6 +196,31 @@ pub(crate) fn relay_extension_sites(cwd: &Path) -> Vec { sites } +/// The path `nemo-relay run --agent pi` hands to `-e`, when there is one. +/// +/// Explicit first, then user scope -- the order `relay_extension_sites` already +/// returns. **Project scope is excluded on purpose.** `-e` is not trust-gated, so +/// promoting a project-scoped extension to it would run repository code pi itself +/// declined to trust: the launcher would be undoing the very gate this module +/// exists to report. A site that is not a path on disk is excluded for a related +/// reason -- `pi install` can record an npm or git specifier, and pi resolves an +/// `-e` argument as a package *source*, so handing one back could make a launch +/// fetch and install from the network. +/// +/// Handing pi a path it would have discovered anyway is safe: pi canonicalizes and +/// de-duplicates the merged command-line and discovered sets before loading +/// (`mergePaths`, pi `v0.84.0`, `core/resource-loader.ts:845`), and both routes +/// resolve a package directory through the same `pi.extensions` manifest, so the +/// extension loads -- and registers its hooks -- exactly once. Passing the +/// directory is also why nothing here reads that manifest: pi does it, and its +/// entry-point precedence is pi's to change. +pub(crate) fn launchable_extension_path(cwd: &Path) -> Option { + relay_extension_sites(cwd) + .into_iter() + .find(|site| site.scope != ExtensionScope::Project && site.path.exists()) + .map(|site| site.path) +} + /// The NeMo Relay extension inside a pi auto-discovery directory, if it is there. /// /// Matched on the package name, not on "the directory is non-empty". A user with diff --git a/crates/cli/src/agents/pi/launch.rs b/crates/cli/src/agents/pi/launch.rs index f272c8945..63e6470ad 100644 --- a/crates/cli/src/agents/pi/launch.rs +++ b/crates/cli/src/agents/pi/launch.rs @@ -62,8 +62,12 @@ pub(crate) fn prepare( // extension regardless of the user's own pi configuration. let Some(path) = extension_path() else { return Err(CliError::Launch(format!( - "could not locate the NeMo Relay pi extension; set {PI_EXTENSION_PATH_ENV} to its \ - entry point, or install it with `pi install ` and launch pi directly" + "could not locate the NeMo Relay pi extension at a load path that is not \ + trust-gated; install it with `pi install ` (without \ + `--local`), copy it into `~/.pi/agent/extensions/`, or set \ + {PI_EXTENSION_PATH_ENV} to its entry point. A project-scoped install is \ + deliberately not used here: `-e` is never trust-gated, so passing one would load \ + code pi itself would not trust. `nemo-relay doctor pi` reports what was found" ))); }; let rendered = path.display().to_string(); @@ -94,13 +98,15 @@ fn set_env(launch: &mut PreparedAgentLaunch, name: &str, value: &str) { launch.env.push((name.to_string(), value.to_string())); } -/// Resolve the extension entry point, preferring an explicit override. +/// Resolve the extension entry point, from the same places `doctor` looks. /// -/// There is no installed location to fall back on the way Codex and Claude Code -/// have one, because pi extensions live in the user's own configuration -/// directories rather than in a NeMo Relay-managed plugin root. +/// It reads no environment variable of its own, deliberately. A second read is +/// how the two drifted: `doctor` resolved a user-scope install and reported the +/// setup as ready, while launching refused to start because only the variable +/// counted here -- and the variable is not something any document tells a user +/// to set. `launchable_extension_path` also excludes the project scope, which +/// `-e` would load past pi's trust gate. fn extension_path() -> Option { - std::env::var_os(PI_EXTENSION_PATH_ENV) - .map(PathBuf::from) - .filter(|path| path.exists()) + let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); + super::doctor::launchable_extension_path(&cwd) } diff --git a/crates/cli/tests/coverage/agents/launcher_tests.rs b/crates/cli/tests/coverage/agents/launcher_tests.rs index fdf949787..6590a19c1 100644 --- a/crates/cli/tests/coverage/agents/launcher_tests.rs +++ b/crates/cli/tests/coverage/agents/launcher_tests.rs @@ -1673,12 +1673,22 @@ fn make_executable(path: &Path) { #[test] fn pi_launch_passes_the_gateway_upstreams_the_extension_redirects_against() { let _guard = current_dir_lock().lock().unwrap(); - let extension = std::env::temp_dir().join("nemo-relay-pi-launch-test-extension.ts"); - std::fs::write(&extension, "export default () => {};").unwrap(); - let _env = EnvScope::set(&[( - crate::agents::pi::launch::PI_EXTENSION_PATH_ENV, - Some(extension.as_os_str()), - )]); + let temp = tempfile::tempdir().unwrap(); + let extension = write_relay_pi_package(&temp.path().join("checkout")); + // Pin pi's agent directory at an empty one: the launcher now falls back to a + // user-scope install, so a developer's own would otherwise decide this result. + let empty_agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&empty_agent_dir).unwrap(); + let _env = EnvScope::set(&[ + ( + crate::agents::pi::launch::PI_EXTENSION_PATH_ENV, + Some(extension.as_os_str()), + ), + ( + crate::agents::pi::doctor::PI_AGENT_DIR_ENV, + Some(empty_agent_dir.as_os_str()), + ), + ]); let resolved = ResolvedConfig { gateway: GatewayConfig { @@ -1725,5 +1735,95 @@ fn pi_launch_passes_the_gateway_upstreams_the_extension_redirects_against() { note.contains("https://integrate.api.nvidia.com/v1"), "the launch note should name the upstream redirection is judged against: {note}" ); - let _ = std::fs::remove_file(&extension); +} + +/// A directory pi resolves as this extension: a manifest naming it, beside an +/// entry point. Returns the entry point, which is what `-e` is pointed at. +fn write_relay_pi_package(dir: &std::path::Path) -> std::path::PathBuf { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write(dir.join("package.json"), r#"{"name": "nemo-relay-pi"}"#).unwrap(); + let entry = dir.join("index.ts"); + std::fs::write(&entry, "export default () => {};").unwrap(); + entry +} + +// The install routes the README documents -- `pi install ` and a file drop +// into `~/.pi/agent/extensions/` -- set no environment variable, and no document +// tells a user to set one. While launching read only that variable, `doctor` +// reported the extension as ready and the launcher refused to start. +#[test] +fn pi_launch_finds_a_user_scope_install_without_an_environment_variable() { + let _guard = current_dir_lock().lock().unwrap(); + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + let installed = agent_dir.join("extensions").join("nemo-relay"); + write_relay_pi_package(&installed); + let _env = EnvScope::set(&[ + (crate::agents::pi::launch::PI_EXTENSION_PATH_ENV, None), + ( + crate::agents::pi::doctor::PI_AGENT_DIR_ENV, + Some(agent_dir.as_os_str()), + ), + ]); + + let prepared = PreparedAgentLaunch::new( + CodingAgent::Pi, + vec!["pi".into()], + "http://127.0.0.1:4040", + &ResolvedConfig::default(), + false, + ) + .unwrap(); + + // pi de-duplicates the merged command-line and discovered extension sets by + // canonical path, so passing `-e` for something it would have found anyway + // loads it once -- and keeps the load working under `--no-extensions`. + let rendered = installed.display().to_string(); + assert!( + prepared + .argv + .windows(2) + .any(|pair| pair[0] == "-e" && pair[1] == rendered), + "a user-scope install must be launchable: {:?}", + prepared.argv + ); +} + +// `-e` is never trust-gated. Promoting a project-scoped install to it would run +// repository code pi itself declined to trust, which is the failure the preflight +// exists to warn about rather than to work around. +#[test] +fn pi_launch_refuses_to_promote_a_project_scoped_install() { + let _guard = current_dir_lock().lock().unwrap(); + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + write_relay_pi_package(&project.join(".pi").join("extensions").join("nemo-relay")); + let empty_agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&empty_agent_dir).unwrap(); + let _env = EnvScope::set(&[ + (crate::agents::pi::launch::PI_EXTENSION_PATH_ENV, None), + ( + crate::agents::pi::doctor::PI_AGENT_DIR_ENV, + Some(empty_agent_dir.as_os_str()), + ), + ]); + let previous = std::env::current_dir().unwrap(); + std::env::set_current_dir(&project).unwrap(); + + let prepared = PreparedAgentLaunch::new( + CodingAgent::Pi, + vec!["pi".into()], + "http://127.0.0.1:4040", + &ResolvedConfig::default(), + false, + ); + + std::env::set_current_dir(previous).unwrap(); + let Err(error) = prepared else { + panic!("a project-scoped install must not be promoted to `-e`"); + }; + assert!( + error.to_string().contains("trust-gated"), + "the launch error should say why the install it can see was not used: {error}" + ); } diff --git a/crates/cli/tests/coverage/agents/pi_doctor_tests.rs b/crates/cli/tests/coverage/agents/pi_doctor_tests.rs index 0160fd9a1..0c48247ed 100644 --- a/crates/cli/tests/coverage/agents/pi_doctor_tests.rs +++ b/crates/cli/tests/coverage/agents/pi_doctor_tests.rs @@ -76,8 +76,9 @@ fn an_empty_project_directory_is_not_reported() { #[test] fn the_explicit_path_is_reported_as_ungated() { let temp = tempfile::tempdir().unwrap(); - let entry = temp.path().join("index.ts"); - std::fs::write(&entry, "export default 1").unwrap(); + let package = temp.path().join("checkout"); + write_relay_package(&package); + let entry = package.join("index.ts"); let empty_home = temp.path().join("home"); std::fs::create_dir_all(&empty_home).unwrap(); @@ -274,13 +275,88 @@ fn somebody_elses_pi_extension_is_not_reported_as_ours() { assert!(!extension_configured()); } +// The explicit route was the one hole in that name check: any path that existed +// counted, so a stale variable made doctor Pass *and* made the launcher inject the +// path with `-e` -- a green check describing a session with no Relay code in it. +#[test] +fn an_unrelated_extension_named_by_the_environment_is_not_reported_as_ours() { + let temp = tempfile::tempdir().unwrap(); + let other = temp.path().join("other"); + write_other_package(&other); + let empty_home = temp.path().join("home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let _env = scoped( + Some(other.join("index.ts").as_os_str()), + Some(empty_home.as_os_str()), + ); + assert!(relay_extension_sites(temp.path()).is_empty()); + assert!(!extension_configured()); + assert!(launchable_extension_path(temp.path()).is_none()); +} + +// `-e` is never trust-gated, so falling back to a project-scoped site would make +// the launcher run code pi refused to load -- undoing the gate this module reports. +#[test] +fn the_launch_path_never_promotes_a_project_scoped_extension() { + let temp = tempfile::tempdir().unwrap(); + let project_extensions = temp.path().join(".pi").join("extensions"); + std::fs::create_dir_all(&project_extensions).unwrap(); + write_relay_package(&project_extensions.join("nemo-relay")); + let empty_home = temp.path().join("home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let _env = scoped(None, Some(empty_home.as_os_str())); + assert!(!relay_extension_sites(temp.path()).is_empty()); + assert!(launchable_extension_path(temp.path()).is_none()); +} + +// pi resolves an `-e` argument as a package source, so an npm specifier that +// `pi install` recorded would be fetched and installed by a launch, not loaded. +#[test] +fn the_launch_path_skips_an_installed_source_that_is_not_a_path() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + std::fs::write( + agent_dir.join("settings.json"), + r#"{"packages": ["npm:nemo-relay-pi"]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + assert!(!relay_extension_sites(temp.path()).is_empty()); + assert!(launchable_extension_path(temp.path()).is_none()); +} + +// A user-scope install is what the README's install routes produce, and none of +// them set an environment variable -- so the launcher has to find one without it. +// An explicit path still wins, because someone who set it meant it. +#[test] +fn the_launch_path_prefers_the_explicit_route_over_a_user_scope_install() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + let installed = agent_dir.join("extensions").join("nemo-relay"); + write_relay_package(&installed); + + let _discovered = scoped(None, Some(agent_dir.as_os_str())); + assert_eq!(launchable_extension_path(temp.path()), Some(installed)); + drop(_discovered); + + let explicit = temp.path().join("checkout"); + write_relay_package(&explicit); + let _env = scoped(Some(explicit.as_os_str()), Some(agent_dir.as_os_str())); + assert_eq!(launchable_extension_path(temp.path()), Some(explicit)); +} + // The headline status and the load-path check must agree: they now answer from the // same resolution, so one `doctor` run cannot say "not located" beside a Pass. #[test] fn the_headline_status_agrees_with_the_load_path_check() { let temp = tempfile::tempdir().unwrap(); - let entry = temp.path().join("index.ts"); - std::fs::write(&entry, "export default 1").unwrap(); + let package = temp.path().join("checkout"); + write_relay_package(&package); + let entry = package.join("index.ts"); let empty_home = temp.path().join("home"); std::fs::create_dir_all(&empty_home).unwrap(); diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index 226fb7fb4..e3c5a023f 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -114,10 +114,15 @@ trust-ungated, loads before extension discovery, and survives `--no-extensions`, so a launched session is instrumented regardless of the user's own pi configuration. -Set `NEMO_RELAY_PI_EXTENSION` to the extension entry point. Unlike Claude Code -and Codex, there is no Relay-managed install location to fall back on, because -pi extensions live in the user's own configuration directories. Launch fails -with that instruction when the extension cannot be located. +The launcher resolves the extension from the same places `nemo-relay doctor pi` +looks — `NEMO_RELAY_PI_EXTENSION` first, then a user-scope install — so either +install route below is enough and no variable is required. Unlike Claude Code and +Codex, there is no Relay-managed install location to fall back on, because pi +extensions live in the user's own configuration directories. + +A **project-scoped** install is deliberately never used. `-e` is not trust-gated, +so passing one would load repository code pi itself declined to trust. Launch +fails instead, and names the install routes that are not gated. Inspect what would be launched without starting pi: diff --git a/integrations/pi/README.md b/integrations/pi/README.md index 004f416f0..33e686fcd 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -61,6 +61,12 @@ pi install /path/to/NeMo-Relay/integrations/pi | `.pi/extensions/` or `pi install --local` | **No** | **Yes** | | `pi install ` | **No** — see below | — | +Either user-scope route is enough for `nemo-relay run --agent pi`: the launcher +resolves the extension from the same places `nemo-relay doctor pi` looks and +passes what it finds to `-e`, so no environment variable is needed. It never +promotes a **project-scoped** install that way — `-e` is not trust-gated, so +doing so would load code pi itself declined to trust. + ⚠️ **A git URL does not work, and fails silently.** pi has no subdirectory syntax for a git source: it clones the repository *root*, then looks there for a `pi` key in `package.json` or a top-level `extensions/` directory. @@ -99,6 +105,7 @@ Run it first whenever Relay does not seem to be doing anything. | Variable | Default | Meaning | |---|---|---| +| `NEMO_RELAY_PI_EXTENSION` | unset | Overrides where the launcher looks for this extension. Set by the launcher from what it resolved | | `NEMO_RELAY_PI_GATEWAY_URL` | `http://127.0.0.1:4040` | Gateway base URL | | `NEMO_RELAY_PI_TIMEOUT_MS` | `5000` | Per-request timeout | | `NEMO_RELAY_PI_FAIL` | `open` | `closed` blocks tool calls and inline shell commands when the gateway is unreachable | From e15aa27eb9091e1aa2fca549bfff27068f0fcc53 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 09:34:58 -0700 Subject: [PATCH 22/41] fix(pi): publish an argument rewrite only once the tool start cannot fail A review asked for `tool_conditional_execution` to be re-run on the transformed arguments, on the ground that an intercept can rewrite a policy-sensitive value after the guardrail allowed the call. The sequence is real, but that is the runtime's documented managed-call order -- the core implements it identically in `tool_call_execute` -- so re-deciding here would fork the middleware contract for one harness, evaluate every conditional guardrail twice, emit two guardrail scope pairs per call, and bill a counting or LLM-judge guardrail twice. The order is now stated where it is load-bearing, in the code and in both documents, rather than changed. What the same read did surface is an ordering hazard that was live. `tool_argument_transform` was assigned before the fallible `tool_call(...)`, and the field lives on the session until a hook response drains it. A rewrite recorded before a failing start therefore rode out on the *next* response, where the extension's `tool_call_id` echo reads it as another call's rewrite and refuses that call. It is published after the start can no longer fail. Tests: the transform test now registers a second, later intercept, so the break-chain flag its comment justifies is actually exercised -- it fails if the chain stops early. A new test pins one guardrail evaluation per call, so a re-check cannot land silently. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- crates/cli/src/sessions/mod.rs | 16 ++- .../cli/tests/coverage/shared/server_tests.rs | 101 +++++++++++++++++- docs/nemo-relay-cli/pi.mdx | 5 + integrations/pi/README.md | 6 ++ 4 files changed, 126 insertions(+), 2 deletions(-) diff --git a/crates/cli/src/sessions/mod.rs b/crates/cli/src/sessions/mod.rs index 5bd95ba0b..56a40195c 100644 --- a/crates/cli/src/sessions/mod.rs +++ b/crates/cli/src/sessions/mod.rs @@ -1566,11 +1566,20 @@ impl Session { // `AgentKind::applies_tool_argument_transforms`. The transformed arguments become the // span's arguments too, so the trace records what will execute rather than what was // proposed. + // + // The verdict above is on the arguments the harness proposed, and the chain is + // deliberately not re-run on the rewrite. That is the order a managed `tool_call_execute` + // uses, so a hook-driven tool call is gated exactly as an in-process one is. An intercept + // is policy, not something policy has to defend against -- re-deciding here would fork + // the middleware contract for one harness, and would evaluate every conditional guardrail + // twice, emitting two guardrail scope pairs and billing an LLM-judge guardrail twice for + // one call. + let mut rewrite = None; let arguments = if self.agent_kind.applies_tool_argument_transforms() { let transformed = tool_request_intercepts(event.tool_name.as_str(), arguments.clone()).await?; if transformed != arguments { - self.tool_argument_transform = Some(ToolArgumentTransform { + rewrite = Some(ToolArgumentTransform { tool_call_id: event.tool_call_id.clone(), arguments: transformed.clone(), }); @@ -1609,6 +1618,11 @@ impl Session { owner_subagent_id: active_tool_owner_subagent_id, }, ); + // Published only once the start cannot fail. The field lives on the session until a hook + // response drains it, so a rewrite recorded before a failing start would ride out on the + // *next* response instead -- where the extension's `tool_call_id` echo reads it as + // another call's rewrite and refuses that call. + self.tool_argument_transform = rewrite; Ok(()) } diff --git a/crates/cli/tests/coverage/shared/server_tests.rs b/crates/cli/tests/coverage/shared/server_tests.rs index abf765339..cabe70878 100644 --- a/crates/cli/tests/coverage/shared/server_tests.rs +++ b/crates/cli/tests/coverage/shared/server_tests.rs @@ -4666,6 +4666,29 @@ async fn pi_tool_call_hook_returns_arguments_a_request_intercept_rewrote() { .unwrap(); let _cleanup = ToolInterceptCleanup("cli-pi-redactor"); + let _ = deregister_tool_request_intercept("cli-pi-chain-witness"); + register_tool_request_intercept( + "cli-pi-chain-witness", + // A higher number runs later: this is the "later intercept" the flag above protects, and + // it fires only on the first rewrite's output, so a chain that stopped early shows up in + // the response body as the unrewritten path. + 2, + false, + Arc::new(|_name: String, args: Value| { + Box::pin(async move { + let mut args = args; + if let Some(object) = args.as_object_mut() + && object.get("path").and_then(Value::as_str) == Some("/work/.env.example") + { + object.insert("path".into(), json!("/work/.env.sample")); + } + Ok(args) + }) + }), + ) + .unwrap(); + let _witness_cleanup = ToolInterceptCleanup("cli-pi-chain-witness"); + let app = router(test_config()); let response = app .oneshot( @@ -4692,9 +4715,85 @@ async fn pi_tool_call_hook_returns_arguments_a_request_intercept_rewrote() { let bytes = response.into_body().collect().await.unwrap().to_bytes(); let body: Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(body["tool_call"]["tool_call_id"], json!("call-transform")); + // Only reachable through both intercepts in order, so this pins that the hook response + // carries the end of the chain rather than the first rewrite. assert_eq!( body["tool_call"]["input"], - json!({ "path": "/work/.env.example" }) + json!({ "path": "/work/.env.sample" }) + ); +} + +// The verdict is on the arguments pi proposed. Pinning one evaluation per call is what keeps the +// pi hook on the same order as a managed tool call, and keeps a counting or LLM-judge guardrail +// from being asked -- and billed -- twice about one call just because an intercept rewrote it. +#[tokio::test] +async fn pi_tool_call_hook_evaluates_conditional_guardrails_once_when_an_intercept_rewrites() { + let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; + static EVALUATIONS: AtomicUsize = AtomicUsize::new(0); + EVALUATIONS.store(0, Ordering::SeqCst); + + let _ = deregister_tool_conditional_execution_guardrail("cli-pi-transform-counter"); + register_tool_conditional_execution_guardrail( + "cli-pi-transform-counter", + 1, + Arc::new(|_name, _args| { + Box::pin(async move { + EVALUATIONS.fetch_add(1, Ordering::SeqCst); + Ok(None) + }) + }), + ) + .unwrap(); + let _guardrail_cleanup = ToolGuardrailCleanup("cli-pi-transform-counter"); + + let _ = deregister_tool_request_intercept("cli-pi-transform-counter-intercept"); + register_tool_request_intercept( + "cli-pi-transform-counter-intercept", + 1, + false, + Arc::new(|_name: String, args: Value| { + Box::pin(async move { + let mut args = args; + if let Some(object) = args.as_object_mut() { + object.insert("path".into(), json!("/work/.env")); + } + Ok(args) + }) + }), + ) + .unwrap(); + let _intercept_cleanup = ToolInterceptCleanup("cli-pi-transform-counter-intercept"); + + let app = router(test_config()); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/hooks/pi") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "session_id": "pi-transform-counter-session", + "hook_event_name": "tool_call", + "tool_call_id": "call-counted", + "tool_name": "read", + "input": { "path": "/work/README.md" } + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["tool_call"]["input"], json!({ "path": "/work/.env" })); + assert_eq!( + EVALUATIONS.load(Ordering::SeqCst), + 1, + "the conditional chain must decide once, on the arguments pi proposed" ); } diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index e3c5a023f..fab5a7939 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -207,6 +207,11 @@ may not add or remove keys, change a type, or change an array's length. That keeps the required keys and types intact, but it is not schema validation — `pattern`, `enum` and range constraints are not checked. A transform that rewrites a value to one the schema would reject will still execute. + +Conditional-execution guardrails decide on the arguments pi proposed, before the +rewrite, and are not re-run on the result — the same order a managed tool call +uses. A request intercept can therefore rewrite a value a guardrail would have +refused. Put the decision in the guardrail, not in a transform that outruns it. A transform that violates the invariant **blocks the call**, with a reason diff --git a/integrations/pi/README.md b/integrations/pi/README.md index 33e686fcd..a1c45ae16 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -170,6 +170,12 @@ the required keys of the required types afterwards. `format` are not checked — by choice, not because the schema is out of reach. A transform that rewrites a string to one the schema would reject still executes. +**Nor is it a second policy decision.** The gateway's conditional-execution +guardrails decide on the arguments pi proposed, before the rewrite, and are not +re-run on the result — the same order a managed tool call uses. A request +intercept can therefore rewrite a value a guardrail would have refused. Put the +decision in the guardrail, not in a transform that outruns it. + A refused transform **blocks the call**, with a reason that says the policy could not be applied rather than that the request was refused. Running the original arguments instead would silently discard a policy decision, which is the failure From 540b471c3aa0536afcbfca279ad8f7d3767b89cc Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 09:37:13 -0700 Subject: [PATCH 23/41] test(pi): exercise the three surfaces this PR added but left unasserted Each of these compiled and passed without touching the thing it was named for. - `route_event_through_alias_covers_all_event_variants` listed twelve of the thirteen `NormalizedEvent` variants. Widening the trailing match arm for `TurnStarted` kept it compiling, so alias rewriting for the one variant this PR added went unverified -- including that it must *not* close the alias. - The centralized version boundaries enumerated Claude Code and Codex only. pi prints a bare semver with no product token, which is the accept path neither of the others exercises, and the reject loop covered only malformed input. - `AgentInfo::checks` is the only carrier of pi's preflight findings, and nothing asserted it -- not the serialized shape, not the human render. The JSON test now pins both halves of `skip_serializing_if`: absent for an agent with no findings, and the nested `{name, status, details}` shape for pi. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- .../tests/coverage/agents/alignment_tests.rs | 4 ++ .../coverage/agents/coding_agent_tests.rs | 8 +++ .../cli/tests/coverage/shared/doctor_tests.rs | 69 ++++++++++++++++--- 3 files changed, 71 insertions(+), 10 deletions(-) diff --git a/crates/cli/tests/coverage/agents/alignment_tests.rs b/crates/cli/tests/coverage/agents/alignment_tests.rs index 286277744..f097f06e0 100644 --- a/crates/cli/tests/coverage/agents/alignment_tests.rs +++ b/crates/cli/tests/coverage/agents/alignment_tests.rs @@ -453,6 +453,10 @@ fn route_event_through_alias_covers_all_event_variants() { let cases = vec![ NormalizedEvent::AgentStarted(session_event("child", "SessionStart")), NormalizedEvent::AgentEnded(session_event("child", "SessionEnd")), + // Only pi reports a turn start, and unlike `TurnEnded` it must not close the alias -- + // there is nothing for the parent to finish on it. Routing never looks at the agent + // kind, so the shared fixture still exercises the arm. + NormalizedEvent::TurnStarted(session_event("child", "turn_start")), NormalizedEvent::TurnEnded(session_event("child", "Stop")), NormalizedEvent::PromptSubmitted(session_event("child", "Prompt")), NormalizedEvent::Compaction(session_event("child", "Compact")), diff --git a/crates/cli/tests/coverage/agents/coding_agent_tests.rs b/crates/cli/tests/coverage/agents/coding_agent_tests.rs index e55a97188..d06bdb27a 100644 --- a/crates/cli/tests/coverage/agents/coding_agent_tests.rs +++ b/crates/cli/tests/coverage/agents/coding_agent_tests.rs @@ -42,6 +42,9 @@ fn centralized_minimum_versions_accept_stable_boundaries() { let cases = [ (CodingAgent::ClaudeCode, "2.1.121 (Claude Code)"), (CodingAgent::Codex, "codex-cli 0.143.0"), + // pi prints the bare version and nothing else, so there is no product token to match -- + // an accept path neither of the others exercises. + (CodingAgent::Pi, "0.84.0"), ]; for (agent, output) in cases { @@ -60,6 +63,11 @@ fn centralized_minimum_versions_reject_old_prerelease_and_malformed_output() { (CodingAgent::ClaudeCode, "2.1.121 (Other Agent)"), (CodingAgent::Codex, "codex-cli 0.142.9"), (CodingAgent::Codex, "codex-cli 0.143.0-alpha.1"), + (CodingAgent::Pi, "0.83.9"), + (CodingAgent::Pi, "0.84.0-alpha.1"), + // A prefixed line is not something pi emits, so it is a parse failure rather than an + // old-version rejection. + (CodingAgent::Pi, "pi 0.84.0"), ]; for (agent, output) in cases { diff --git a/crates/cli/tests/coverage/shared/doctor_tests.rs b/crates/cli/tests/coverage/shared/doctor_tests.rs index 6f98320bf..65bce3289 100644 --- a/crates/cli/tests/coverage/shared/doctor_tests.rs +++ b/crates/cli/tests/coverage/shared/doctor_tests.rs @@ -348,6 +348,20 @@ fn format_human_uses_symbols_for_agent_statuses() { annotation: "not configured".into(), checks: Vec::new(), }, + AgentInfo { + name: "pi", + status: Status::Warn, + configured: true, + command: "pi".into(), + path: Some(PathBuf::from("/bin/pi")), + version: Some("0.84.0".into()), + annotation: "hooks: emitted by the extension".into(), + checks: vec![Check { + name: "pi extension load path", + status: Status::Warn, + details: "project scope, which pi loads only for a trusted project".into(), + }], + }, ]; let rendered = format_human(&report); @@ -356,6 +370,12 @@ fn format_human_uses_symbols_for_agent_statuses() { assert!(rendered.contains(" · codex")); assert!(!rendered.contains(" pass ")); assert!(!rendered.contains(" info ")); + // pi's findings are the reason agents carry checks at all: where the extension sits decides + // whether pi loads it, and a status symbol alone cannot say that. + assert!( + rendered.contains("pi extension load path: project scope"), + "an agent check must reach the human report: {rendered}" + ); } #[test] @@ -2405,16 +2425,32 @@ fn format_agents_human_lists_supported_and_separates_detected() { #[test] fn format_agents_json_matches_doctor_agents_shape() { - let agents = vec![AgentInfo { - name: "claude", - status: Status::Pass, - configured: true, - command: "claude".into(), - path: Some(PathBuf::from("/opt/homebrew/bin/claude")), - version: Some("2.1.4".into()), - annotation: "hooks: injected during run".into(), - checks: Vec::new(), - }]; + let agents = vec![ + AgentInfo { + name: "claude", + status: Status::Pass, + configured: true, + command: "claude".into(), + path: Some(PathBuf::from("/opt/homebrew/bin/claude")), + version: Some("2.1.4".into()), + annotation: "hooks: injected during run".into(), + checks: Vec::new(), + }, + AgentInfo { + name: "pi", + status: Status::Warn, + configured: true, + command: "pi".into(), + path: Some(PathBuf::from("/opt/homebrew/bin/pi")), + version: Some("0.84.0".into()), + annotation: "hooks: emitted by the extension".into(), + checks: vec![Check { + name: "pi extension load path", + status: Status::Warn, + details: "project scope, which pi loads only for a trusted project".into(), + }], + }, + ]; let json = format_agents_json(&agents).unwrap(); let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); assert!(parsed.is_array()); @@ -2424,6 +2460,19 @@ fn format_agents_json_matches_doctor_agents_shape() { assert_eq!(parsed[0]["command"], "claude"); assert_eq!(parsed[0]["version"], "2.1.4"); assert_eq!(parsed[0]["path"], "/opt/homebrew/bin/claude"); + // An agent with no findings must not grow an empty array in the schema: the field is skipped, + // so a consumer written against the pre-pi shape stays valid. + assert!(parsed[0].get("checks").is_none()); + // pi is the one agent that reports findings of its own, so the nested shape is part of the + // published schema rather than an implementation detail. + assert_eq!( + parsed[1]["checks"], + serde_json::json!([{ + "name": "pi extension load path", + "status": "warn", + "details": "project scope, which pi loads only for a trusted project" + }]) + ); } #[test] From 836e76b3d2e8befbf6aa60f7052e22f55cf5f54c Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 09:39:53 -0700 Subject: [PATCH 24/41] fix(pi): close a lookup that fails open, and name the redirect skip honestly Two defects in the same decision path, both found by re-reading it rather than by a failing test. `SERVICEABLE_APIS` was an object literal, and pi types `api` as `KnownApi | (string & {})` -- a free-form string out of models.json, remote catalogs and other extensions. A model whose `api` is `constructor`, `toString`, `valueOf` or `__proto__` therefore resolved through the prototype chain to something truthy, the unserviceable-API guard did not fire, and the model was scored against the Anthropic upstream and redirected into a route the gateway does not have -- with the inherited value rendered into the reason string. A `Map` has no prototype keys to inherit. `noUncheckedIndexedAccess` already typed the lookup as optional, so no call site changes. The `upstream-mismatch` skip was effectively unreachable. `siblingsOf` reads `ctx.modelRegistry.getAll()`, which returns the selected model too, so with a real pi runtime a model is always one of its own siblings -- and the whole-provider scan ran first. The commonest outcome there is, an ordinary endpoint mismatch, was reported as `provider-mixed-endpoints` naming the selected model as the sibling blocking itself. The selected model's own check now runs first. Same decision either way; the code an operator reads is now the one they can act on. Tests: both cases fail against the previous source. The stub gateway gains a `raw` reply so a case can put a body on the wire that `JSON.parse` cannot read -- the inline-shell suite's "unparseable success body" condition was sending a valid `{}` and passing as a plain allow, so the fault branch it was named for was never reached. Two comments claimed pi keeps the extension runtime alive across `/new`, `/resume` and `/fork`. It does not; it rebuilds it. The defensive re-registration stays, described as the defence it is. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- integrations/pi/index.ts | 8 ++- integrations/pi/src/provider-redirect.ts | 55 ++++++++++++------- integrations/pi/test/harness.mjs | 10 +++- integrations/pi/test/lifecycle.test.mjs | 3 +- .../pi/test/provider-redirect.test.mjs | 29 ++++++++++ integrations/pi/test/user-bash.test.mjs | 20 ++++++- 6 files changed, 98 insertions(+), 27 deletions(-) diff --git a/integrations/pi/index.ts b/integrations/pi/index.ts index 3e02b1f0e..17dabe35a 100644 --- a/integrations/pi/index.ts +++ b/integrations/pi/index.ts @@ -248,9 +248,11 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { redirect ??= redirectConfigFromEnv(active.url); // The join key is baked into the registration, so it cannot follow a session - // replacement on its own: `/new`, `/resume` and `/fork` keep this runtime - // alive and give it a new session id, leaving every redirected provider - // stamping the old one onto its requests. Re-register when it moves. + // replacement on its own. pi `v0.84.0` tears the extension runtime down and + // builds a fresh one for `/new`, `/resume` and `/fork`, so this branch does not + // fire there -- it is kept for a host that reuses one runtime across session + // ids, which would otherwise leave every redirected provider stamping the old + // one onto its requests. Re-register when it moves. const sessionKey = safeSessionId(ctx); if (registeredSessionKey !== null && registeredSessionKey !== sessionKey) { redirectedProviders.clear(); diff --git a/integrations/pi/src/provider-redirect.ts b/integrations/pi/src/provider-redirect.ts index 57bdf2329..6ad166241 100644 --- a/integrations/pi/src/provider-redirect.ts +++ b/integrations/pi/src/provider-redirect.ts @@ -32,12 +32,22 @@ * endpoint and produce no LLM spans, which is the honest outcome. */ -/** The API families the gateway serves, mapped to the upstream that backs each. */ -const SERVICEABLE_APIS: Record = { - 'openai-completions': 'openai', - 'openai-responses': 'openai', - 'anthropic-messages': 'anthropic', -}; +/** + * The API families the gateway serves, mapped to the upstream that backs each. + * + * A `Map` rather than an object literal because pi does not constrain this key: + * `api` is `KnownApi | (string & {})` and reaches us from models.json, remote + * catalogs and other extensions. On an object literal an `api` of `constructor` + * or `__proto__` resolves through the prototype chain to something truthy, so + * the unserviceable-API guard -- whose whole job is to keep a model the gateway + * cannot route out of the gateway -- would not fire, and the model was scored + * against the Anthropic upstream instead. + */ +const SERVICEABLE_APIS = new Map([ + ['openai-completions', 'openai'], + ['openai-responses', 'openai'], + ['anthropic-messages', 'anthropic'], +]); export type RedirectConfig = { /** Gateway base URL; the root, never root + `/v1`. */ @@ -92,7 +102,7 @@ export type RedirectDecision = * to. Applied to every sibling because the registration is provider-wide. */ function isSafeToRedirect(model: RedirectModel, config: RedirectConfig): boolean { - const family = SERVICEABLE_APIS[model.api]; + const family = SERVICEABLE_APIS.get(model.api); if (!family) return false; const upstream = family === 'openai' ? config.openaiUpstream : config.anthropicUpstream; if (!upstream) return false; @@ -164,7 +174,7 @@ export function decideRedirect( }; } - const family = SERVICEABLE_APIS[model.api]; + const family = SERVICEABLE_APIS.get(model.api); if (!family) { // Seven of pi's 39 built-in providers speak an API the gateway has no // route for: Bedrock, Azure OpenAI Responses, Google, Google Vertex, @@ -210,6 +220,23 @@ export function decideRedirect( }; } + // Ahead of the sibling scan, deliberately. pi's `modelRegistry.getAll()` returns + // the selected model too, so it is one of its own siblings -- and while the scan + // ran first, an ordinary endpoint mismatch was reported as + // `provider-mixed-endpoints`, naming the selected model as the sibling that + // blocked it. Same decision either way; this is the code an operator can act on. + if (normalizeBaseUrl(upstream) !== normalizeBaseUrl(model.baseUrl)) { + return { + kind: 'skip', + code: 'upstream-mismatch', + provider: model.provider, + api: model.api, + reason: + `model targets ${model.baseUrl} but the gateway forwards ${family} traffic to ${upstream}; ` + + `redirecting would send the request to the wrong provider`, + }; + } + // `registerProvider(name, {baseUrl})` rewrites the URL of EVERY model of that // provider, so a decision made from the selected model alone is a decision made // on behalf of its siblings. Several pi 0.84 providers mix API families at @@ -232,18 +259,6 @@ export function decideRedirect( }; } - if (normalizeBaseUrl(upstream) !== normalizeBaseUrl(model.baseUrl)) { - return { - kind: 'skip', - code: 'upstream-mismatch', - provider: model.provider, - api: model.api, - reason: - `model targets ${model.baseUrl} but the gateway forwards ${family} traffic to ${upstream}; ` + - `redirecting would send the request to the wrong provider`, - }; - } - return { kind: 'redirect', provider: model.provider, diff --git a/integrations/pi/test/harness.mjs b/integrations/pi/test/harness.mjs index a0b690b68..569dfb185 100644 --- a/integrations/pi/test/harness.mjs +++ b/integrations/pi/test/harness.mjs @@ -29,6 +29,9 @@ import { createServer } from 'node:http'; * post the block came from. * * @param gatedHook the `hook_event_name` whose reply `replyWith` controls + * + * A reply is `{status, payload, delayMs}`, or `{status, raw, delayMs}` to send a + * body verbatim -- which is the only way to reach the unreadable-success path. */ export function stubGateway(gatedHook) { const posts = []; @@ -42,10 +45,13 @@ export function stubGateway(gatedHook) { const parsed = JSON.parse(body || '{}'); posts.push(parsed); const gated = gatedHook !== undefined && parsed.hook_event_name === gatedHook; - const { status, payload, delayMs } = gated ? reply : { status: 200, payload: {} }; + const { status, payload, raw, delayMs } = gated ? reply : { status: 200, payload: {} }; const send = () => { res.writeHead(status, { 'content-type': 'application/json' }); - res.end(JSON.stringify(payload ?? {})); + // `raw` lets a case put a body on the wire that `JSON.parse` cannot read. + // The client treats an unreadable success as a fault rather than an empty + // allow, and nothing `JSON.stringify` produces can reach that branch. + res.end(raw ?? JSON.stringify(payload ?? {})); }; if (delayMs) setTimeout(send, delayMs); else send(); diff --git a/integrations/pi/test/lifecycle.test.mjs b/integrations/pi/test/lifecycle.test.mjs index a65357a6a..8ce37eeb2 100644 --- a/integrations/pi/test/lifecycle.test.mjs +++ b/integrations/pi/test/lifecycle.test.mjs @@ -666,7 +666,8 @@ describe('the session join key on redirected providers', () => { }); // The key is baked in at registration, so it cannot follow a replacement on its - // own -- `/new`, `/resume` and `/fork` keep this runtime alive with a new id. + // own. pi `v0.84.0` rebuilds the runtime for `/new`, `/resume` and `/fork`, so + // this pins the defence rather than a path pi takes today. it('is refreshed when the session is replaced under the same runtime', async () => { const { fire, registrations, setSession } = loadRecording('sess-one'); await fire('session_start', { reason: 'startup' }); diff --git a/integrations/pi/test/provider-redirect.test.mjs b/integrations/pi/test/provider-redirect.test.mjs index 34d530788..8c408b133 100644 --- a/integrations/pi/test/provider-redirect.test.mjs +++ b/integrations/pi/test/provider-redirect.test.mjs @@ -66,6 +66,22 @@ describe('redirect decision', () => { assert.match(decision.reason, /wrong provider/); }); + // pi's `modelRegistry.getAll()` returns the selected model too, so with a real pi + // runtime a model is always one of its own siblings. While the whole-provider scan + // ran first, this -- the ordinary mismatch, and the commonest outcome there is -- + // was reported as `provider-mixed-endpoints`, naming the selected model as the + // sibling blocking itself. Same decision; the code has to be the actionable one. + it('reports an ordinary mismatch as itself when the model is its own sibling', () => { + const decision = decideRedirect( + nvidiaModel, + matchConfig({ openaiUpstream: 'https://api.openai.com/v1' }), + none, + [nvidiaModel], + ); + assert.equal(decision.kind, 'skip'); + assert.equal(decision.code, 'upstream-mismatch'); + }); + it('picks the upstream matching the model’s API family, not the other one', () => { // Anthropic model, anthropic upstream matches, openai upstream does not. const decision = decideRedirect( @@ -85,6 +101,19 @@ describe('redirect decision', () => { } }); + // `model.api` is a free-form string in pi, so the serviceable-API lookup must not + // answer for names every JavaScript object inherits. On an object-literal map + // `constructor` and `__proto__` read back truthy, and this model -- whose endpoint + // happens to be the anthropic upstream -- was redirected into a route the gateway + // does not have, with the inherited value rendered into the reason string. + it('refuses an API named after an inherited object property', () => { + for (const api of ['constructor', 'toString', 'valueOf', '__proto__']) { + const decision = decideRedirect({ ...anthropicModel, api }, matchConfig(), none); + assert.equal(decision.kind, 'skip', api); + assert.equal(decision.code, 'unserviceable-api', api); + } + }); + // Launched outside `nemo-relay run --agent pi`, so nothing told the extension // what the gateway fronts. Staying put costs spans; guessing costs the session. it('refuses when the gateway upstream is unknown', () => { diff --git a/integrations/pi/test/user-bash.test.mjs b/integrations/pi/test/user-bash.test.mjs index e8fffb9ed..bd6757f1b 100644 --- a/integrations/pi/test/user-bash.test.mjs +++ b/integrations/pi/test/user-bash.test.mjs @@ -170,6 +170,24 @@ describe('inline shell gate', () => { } }); + // A 2xx body that does not parse may have carried a required transform, so it + // cannot be read as an empty allow -- that runs the original command and discards + // the policy. Only a raw body reaches this branch; anything `JSON.stringify` + // produces parses, which is why the adverse-condition sweep below could not see it. + it('treats a success body it cannot read as a fault, not an empty allow', async () => { + process.env.NEMO_RELAY_PI_FAIL = 'closed'; + gateway.replyWith({ status: 200, raw: '{ truncated' }); + const fire = load(); + const result = await fire('user_bash', { + command: 'ls', + excludeFromContext: false, + cwd: '/work', + }); + assert.ok(result?.result, 'an unreadable success must not fall through to an allow'); + assert.equal(result.result.exitCode, REFUSED_EXIT_CODE); + assert.match(result.result.output, /infrastructure fault, not a judgement/); + }); + it('forwards the !! form so a policy can see the output will bypass the model', async () => { const fire = load(); await fire('user_bash', { command: 'cat .env', excludeFromContext: true, cwd: '/work' }); @@ -186,7 +204,7 @@ describe('inline shell gate', () => { ['gateway error', { status: 500, payload: { error: { message: 'kaboom' } } }, url], ['403 without the guardrail marker', { status: 403, payload: { error: {} } }, url], ['malformed rejection body', { status: 403, payload: 'not-an-object' }, url], - ['unparseable success body', { status: 200, payload: undefined }, url], + ['unparseable success body', { status: 200, raw: '{ truncated' }, url], ['unreachable gateway', { status: 200, payload: {} }, 'http://127.0.0.1:1'], // pi builds its terminal component only after this handler resolves, so a gateway // that never answers shows the user nothing at all until the timeout fires. From 9e9e679be9a5428869c1859f6dbc40329bf23469 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 09:40:22 -0700 Subject: [PATCH 25/41] chore(pi): declare the node types this workspace typechecks against `integrations/pi/tsconfig.json` sets `"types": ["node"]` and the sources need it -- `fetch`, `AbortController`, `setTimeout`, `process` and `URL` all come from there. The workspace declared no dependencies at all, so `@types/node` resolved only because the sibling OpenClaw workspace declares it and npm hoists it to the root. Re-spec or drop it there and `just test-pi` breaks in a directory nobody edited. The spec matches OpenClaw's `^24.0.0` exactly, so npm keeps deduping one copy at the root rather than nesting a second under `integrations/pi`. `typescript` is deliberately not added: it resolves from the workspace root by design, which is the difference between the two. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- integrations/pi/package.json | 3 +++ package-lock.json | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/integrations/pi/package.json b/integrations/pi/package.json index 382fb68a5..8e9b7a709 100644 --- a/integrations/pi/package.json +++ b/integrations/pi/package.json @@ -18,5 +18,8 @@ "scripts": { "typecheck": "tsc -p tsconfig.json", "test": "node --test test/*.test.mjs" + }, + "devDependencies": { + "@types/node": "^24.0.0" } } diff --git a/package-lock.json b/package-lock.json index e571cfc3a..d6c176354 100644 --- a/package-lock.json +++ b/package-lock.json @@ -479,7 +479,10 @@ }, "integrations/pi": { "name": "nemo-relay-pi", - "version": "0.8.0" + "version": "0.8.0", + "devDependencies": { + "@types/node": "^24.0.0" + } }, "node_modules/@boundaryml/baml": { "version": "0.219.0", From e716494d348df101f2aee8b75d4ee8084bd2f0b4 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 09:45:48 -0700 Subject: [PATCH 26/41] docs(pi): complete the wire contract, the hook inventory, and the outcomes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six documentation gaps, each verified against the code rather than against the review text that reported it. - **The allow body was documented as always empty.** It can carry `{"tool_call": {…}}`. And the echoed `tool_call_id` is a *precondition*, not decoration: the extension applies a transform only on an exact string match, and refuses -- which blocks -- on a missing, non-string or different id. Neither page said so. - **The hook inventory listed 11 of 15 names.** Missing were `model_redirect`, `tool_arguments_transformed`, `user_bash` and `user_bash_end`. Three of the fifteen are not pi hooks at all, which the row could not convey on its own, so a paragraph names them. The column is keyed by the name the gateway receives, so it carries `model_redirect` rather than pi's `model_select` -- the two tables index differently on purpose. - **`provider-mixed-endpoints` was undocumented** on all five surfaces that describe redirection, along with the reason it exists: `registerProvider` is provider-wide, so the endpoint check is applied to every sibling. - **Guardrail order.** A request intercept can rewrite a value a conditional-execution guardrail would have refused, because the verdict is on the arguments pi proposed. Both documents now say to put the decision in the guardrail. - **Four pi source citations were imprecise.** `package-manager.ts:2394` is a blank line above the guard it describes, `config.ts:515-522` overshoots `getAgentDir` by one, and `extensions/types.ts` is `core/extensions/types.ts` -- pi has both directories. Each now names the pinned version inline, so a pi bump is a grep for `v0.84.0`. - **The pi CI filter comment claimed a coverage it does not have.** `test-pi` runs a TypeScript typecheck and Node tests against a *stub* gateway, so it can never observe the route, the classifier, or the session manager. Those are the `rust` filter's, and listing `crates/cli/src/sessions/**` here would put a full Node matrix on nearly every CLI change for no signal. Also: title case for the README's headings and table headers, matching the repo's documented style and the sibling docs page, with intra-page link text following the headings it names; and en-US throughout the package, including the runtime strings a blocked model reads. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- .github/ci-path-filters.yml | 13 +++-- crates/cli/src/agents/pi/doctor.rs | 11 +++-- docs/nemo-relay-cli/about.mdx | 2 +- docs/nemo-relay-cli/basic-usage.mdx | 21 ++++++-- docs/nemo-relay-cli/pi.mdx | 22 ++++++++- docs/reference/support-matrix.mdx | 2 +- integrations/pi/README.md | 49 +++++++++++-------- integrations/pi/index.ts | 2 +- integrations/pi/src/argument-transform.ts | 4 +- integrations/pi/src/gateway-client.ts | 2 +- integrations/pi/src/pi-hook-types.ts | 21 ++++++-- integrations/pi/src/provider-redirect.ts | 7 +-- integrations/pi/src/user-bash.ts | 6 +-- .../pi/test/argument-transform.test.mjs | 4 +- integrations/pi/test/gateway-client.test.mjs | 4 +- integrations/pi/test/lifecycle.test.mjs | 4 +- integrations/pi/test/tool-call.test.mjs | 6 +-- integrations/pi/test/user-bash.test.mjs | 8 +-- 18 files changed, 125 insertions(+), 63 deletions(-) diff --git a/.github/ci-path-filters.yml b/.github/ci-path-filters.yml index e20d685cf..adabdb232 100644 --- a/.github/ci-path-filters.yml +++ b/.github/ci-path-filters.yml @@ -190,9 +190,16 @@ node: openclaw: - 'integrations/openclaw/**' -# The pi extension is a hook client for the CLI gateway, so its contract is -# shared with the Rust adapter and route -- a change on either side can break -# the other. +# The pi extension is a hook client for the CLI gateway, so a change to the +# adapter it posts against should re-run the extension's own suite. +# +# Only the adapter paths are listed, deliberately. `test-pi` is a TypeScript +# typecheck plus Node tests against a *stub* gateway, so it cannot observe the +# `/hooks/pi` route, the event classifier, or the session manager no matter what +# changes there -- and `crates/cli/src/sessions/**` alone would put a full Node +# matrix on nearly every CLI change. Those files are covered by the `rust` +# filter, which matches every `crates/**/*.rs` file and runs the workspace tests +# that exercise the route for real. pi: - 'crates/cli/src/agents/pi/**' - 'crates/cli/src/agents/shared/adapters.rs' diff --git a/crates/cli/src/agents/pi/doctor.rs b/crates/cli/src/agents/pi/doctor.rs index dd909fc5e..12ef2b257 100644 --- a/crates/cli/src/agents/pi/doctor.rs +++ b/crates/cli/src/agents/pi/doctor.rs @@ -9,9 +9,10 @@ //! extension sits and whether pi will actually load it. //! //! **The failure this module exists for is silent.** pi adds project-scoped -//! extensions to its candidate set only when the project is trusted -//! (`core/package-manager.ts:2394`), and `-p`, `--mode json` and `--mode rpc` -//! never prompt for trust (`docs/security.md:29`). Under the default policy a +//! extensions to its candidate set only when the project is trusted (pi +//! `v0.84.0`, `core/package-manager.ts:2395`), and `-p`, `--mode json` and +//! `--mode rpc` never prompt for trust (`docs/security.md:29`). Under the +//! default policy a //! project-scoped extension is therefore dropped by a bare conditional -- not //! by an error path, so it never reaches pi's extension-load error list and pi //! does not consider it a failure. Nothing reports it, and **the extension @@ -24,8 +25,8 @@ use super::launch::{PI_EXTENSION_PATH_ENV, PI_GATEWAY_URL_ENV}; /// pi's per-user configuration root, `~/.pi/agent` unless overridden. /// -/// Mirrors `getAgentDir()` (pi `config.ts:515-522`), including the environment -/// override, so the preflight looks where pi will actually look. +/// Mirrors `getAgentDir()` (pi `v0.84.0`, `config.ts:515-521`), including the +/// environment override, so the preflight looks where pi will actually look. pub(crate) const PI_AGENT_DIR_ENV: &str = "PI_CODING_AGENT_DIR"; /// pi's configuration directory name, from its `piConfig.configDir`. diff --git a/docs/nemo-relay-cli/about.mdx b/docs/nemo-relay-cli/about.mdx index 646c21a7b..ed8975583 100644 --- a/docs/nemo-relay-cli/about.mdx +++ b/docs/nemo-relay-cli/about.mdx @@ -58,7 +58,7 @@ controls. | --- | --- | --- | --- | --- | | Claude Code | Yes | Yes | Partial | Pre-tool hook responses are supported. LLM optimization uses gateway-routed traffic; full coverage depends on loaded Claude Code hooks. | | Codex | Yes | Yes | Partial | Persistent install verifies all 10 hooks in the supported schema. Each `Stop` finalizes a turn snapshot because the plugin schema does not expose `SessionEnd`. | -| pi | Partial | Yes | No | Proof of concept. Tool and turn activity are captured through a Relay-authored pi extension, and both tool calls and model calls can be blocked. Model traffic reaches the gateway only when the gateway forwards to the endpoint the selected model would otherwise call. | +| pi | Partial | Yes | No | Proof of concept. Tool and turn activity are captured through a Relay-authored pi extension, and both tool calls and model calls can be blocked. Model traffic reaches the gateway only when the gateway forwards to the endpoint every model of the selected provider would otherwise call, because pi's provider registration is provider-wide. | For minimum agent versions, platform support, and current limitations, refer to the [Support Matrix](/reference/support-matrix). diff --git a/docs/nemo-relay-cli/basic-usage.mdx b/docs/nemo-relay-cli/basic-usage.mdx index b68ddccc3..959cb7e05 100644 --- a/docs/nemo-relay-cli/basic-usage.mdx +++ b/docs/nemo-relay-cli/basic-usage.mdx @@ -26,8 +26,11 @@ the payload in a shared gateway envelope. Claude-compatible fields such as `continue` and permission decisions when the hook event supports them. - `POST /hooks/pi` accepts the hook JSON posted by the NeMo Relay pi extension. - It returns an empty object for an allowed call and HTTP 403 with - `error.type = "nemo_relay_guardrail_rejected"` when a guardrail blocks one. + It returns an empty object for an allowed call, or + `{"tool_call": {"tool_call_id": "…", "input": {…}}}` when a request intercept + rewrote the arguments and the extension must execute those instead. It returns + HTTP 403 with `error.type = "nemo_relay_guardrail_rejected"` when a guardrail + blocks one. When a hook closes a turn, subagent, or session scope, Relay returns the hook response after subscribers finish processing that scope-end event. This makes @@ -517,13 +520,23 @@ hints can also add `llm_correlation_source`, `llm_correlation_subagent_id`, `llm_correlation_conversation_id`, `llm_correlation_generation_id`, `llm_correlation_request_id`, and `llm_correlation_agent_type`. -Generated hook bundles subscribe to the events needed for that mapping: +Generated hook bundles subscribe to the events needed for that mapping. pi has +no generated bundle — its extension posts these names directly: | Agent | LLM Lifecycle and Correlation Hooks | Scope, Tool, and Mark Hooks | | --- | --- | --- | | Claude Code | `UserPromptSubmit`, `Stop` | `SessionStart`, `SessionEnd`, `UserPromptExpansion`, `SubagentStart`, `SubagentStop`, `PreToolUse`, `PostToolUse`, `PostToolUseFailure`, `PermissionRequest`, `Notification`, `PreCompact`, `PostCompact` | | Codex | `UserPromptSubmit`, `Stop` | `SessionStart`, `SubagentStart`, `SubagentStop`, `PreToolUse`, `PostToolUse`, `PermissionRequest`, `PreCompact`, `PostCompact` | -| pi | none — pi's LLM spans come from real gateway traffic, not from hooks | `session_start`, `session_shutdown`, `agent_start`, `agent_end`, `agent_settled`, `turn_start`, `turn_end`, `session_before_compact`, `session_compact`, `tool_call`, `tool_execution_end` | +| pi | none — pi's LLM spans come from real gateway traffic, not from hooks | `session_start`, `session_shutdown`, `agent_start`, `agent_end`, `agent_settled`, `turn_start`, `turn_end`, `session_before_compact`, `session_compact`, `model_redirect`, `tool_call`, `tool_arguments_transformed`, `tool_execution_end`, `user_bash`, `user_bash_end` | + +Three of pi's names are not pi hooks. The extension posts `model_redirect` after +deciding whether to point the active model's provider at the gateway, from +`session_start` and from every `model_select`; `tool_arguments_transformed` after +applying a request intercept's rewrite; and `user_bash_end` to close the +inline-shell span, because pi reports no completion for it. pi's own +`tool_execution_start` is deliberately never forwarded — it fires before argument +validation and for calls that never execute, so the extension uses it only to +remember a tool name for the matching end. ## Hook Forwarding diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index fab5a7939..d54a80b02 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -195,6 +195,12 @@ extension applies it to pi's `event.input` in place. | `{}` | Allow, arguments unchanged | | `{"tool_call": {"tool_call_id": "…", "input": {…}}}` | Allow, but execute these arguments | +`tool_call_id` is echoed so the extension can prove the rewrite belongs to the +call it just posted. It applies the transform only on an exact string match. A +body that carries `input` but names a different call, or no call at all, is +refused — and a refused transform blocks, exactly as a shape violation does, +because the alternative is running arguments the policy never approved. + The rewrite is **constrained, not validated**. pi validates tool arguments before the hook and never re-validates. The extension could read the tool's @@ -365,6 +371,16 @@ the extension redirects only on a match: | Gateway forwards elsewhere | Skipped (`upstream-mismatch`) | | Model's API has no gateway route | Skipped (`unserviceable-api`) | | Upstream unknown, e.g. a standalone gateway | Skipped (`unknown-upstream`) | +| Another model of the same provider targets an endpoint the gateway does not front | Skipped (`provider-mixed-endpoints`) | + +`registerProvider` rewrites every model of the provider, not only the selected +one, so the endpoint check is applied to the whole provider. A provider that +mixes endpoints — Fireworks serves `anthropic-messages` at `/inference` and +`openai-completions` at `/inference/v1` — is left alone even when the selected +model matches, because redirecting it would move its siblings to an endpoint that +has never heard of them. Pointing both `--openai-base-url` and +`--anthropic-base-url` at that provider unblocks it, unless a sibling speaks an +API the gateway has no route for at all. Every outcome is recorded as a `model_redirect` mark on the session scope, so a trace with no LLM spans states its own reason. The decision is re-made on each @@ -452,7 +468,9 @@ Look for the `model_redirect` mark on the session scope: it names the outcome an the reason. The common ones are `upstream-mismatch` (start the gateway with `--openai-base-url` or `--anthropic-base-url` pointing at the model's provider), `unknown-upstream` (launch through `nemo-relay run --agent pi`, or set -`NEMO_RELAY_PI_REDIRECT=force`), and `unserviceable-api` (the model's provider -speaks an API the gateway has no route for — pick another model). +`NEMO_RELAY_PI_REDIRECT=force`), `unserviceable-api` (the model's provider +speaks an API the gateway has no route for — pick another model), and +`provider-mixed-endpoints`, whose reason names the sibling model that blocked it +— point both upstreams at that provider, or select a model elsewhere. Tool and turn activity are unaffected by any of these. diff --git a/docs/reference/support-matrix.mdx b/docs/reference/support-matrix.mdx index 2d51db0a7..608e2fac4 100644 --- a/docs/reference/support-matrix.mdx +++ b/docs/reference/support-matrix.mdx @@ -66,7 +66,7 @@ and older CLI versions during installation, diagnostics, and transparent runs. | --- | --- | --- | --- | | Claude Code | 2.1.121 | Persistent install, transparent run, lifecycle hooks, local gateway routing, and pre-tool security | Claude desktop, web, and application sessions are unsupported unless they expose the same local hook and gateway controls. Optimization requires gateway-routed LLM traffic and available hooks. | | Codex CLI | 0.143.0 | Persistent install, transparent run, 10 supported plugin hooks, local gateway routing, and pre-tool security | Cloud or remote tasks that bypass the local machine have partial or no LLM capture. The plugin hook schema has no `SessionEnd`; Relay finalizes the cumulative session snapshot at `Stop`. Encrypted Codex multi-agent v2 payloads cannot be decrypted or reliably linked. | -| pi | 0.84.0 | Transparent run through a Relay-authored pi extension, 15 lifecycle hooks, tool-call security, inline-shell security, and model-call security when redirection applies | Proof of concept. No persistent install: pi has no plugin marketplace. Model traffic is redirected only when the gateway forwards to the endpoint the selected model would otherwise call; otherwise there are no LLM spans for that model. Seven of pi's 39 built-in providers speak an API the gateway has no route for. Subagents and nested pi processes appear as unrelated sessions. pi ships breaking changes through minor releases, so hook signatures need re-verification after an upgrade. | +| pi | 0.84.0 | Transparent run through a Relay-authored pi extension, 15 lifecycle hooks, tool-call security, inline-shell security, and model-call security when redirection applies | Proof of concept. No persistent install: pi has no plugin marketplace. Model traffic is redirected only when the gateway forwards to the endpoint every model of the selected provider would otherwise call, because pi's provider registration is provider-wide; otherwise there are no LLM spans for that model. Seven of pi's 39 built-in providers speak an API the gateway has no route for. Subagents and nested pi processes appear as unrelated sessions. pi ships breaking changes through minor releases, so hook signatures need re-verification after an upgrade. | For installation, diagnostics, and host-specific behavior, refer to [Coding Agent Installation](/nemo-relay-cli/plugin-installation), [Claude diff --git a/integrations/pi/README.md b/integrations/pi/README.md index a1c45ae16..647da4194 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -3,7 +3,7 @@ SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All SPDX-License-Identifier: Apache-2.0 --> -# NeMo Relay extension for pi +# NeMo Relay Extension for pi A pi extension that forwards pi's lifecycle to the NeMo Relay CLI gateway and gates tool calls on the gateway's verdict. @@ -24,7 +24,7 @@ signatures before relying on them. generic environment override — it resolves `baseUrl` per model from a generated catalog — so the extension points the active model's provider at the gateway itself. It only does so when the gateway forwards to the endpoint that -model would otherwise have called; see [Model redirection](#model-redirection). +model would otherwise have called; see [Model Redirection](#model-redirection). When it does not, you get tool and turn activity but no LLM spans. ## Usage @@ -41,7 +41,7 @@ NEMO_RELAY_PI_GATEWAY_URL=http://127.0.0.1:4040 \ `--no-extensions`, which makes it the reliable way to load this. It is also what `nemo-relay run --agent pi` uses. -### Where to install it +### Where to Install It **User scope only**, by either of two routes: @@ -53,7 +53,7 @@ cp -r integrations/pi ~/.pi/agent/extensions/nemo-relay pi install /path/to/NeMo-Relay/integrations/pi ``` -| Path | Install here? | Trust-gated? | +| Path | Install Here? | Trust-Gated? | |---|---|---| | `~/.pi/agent/extensions/` | Yes | No | | `pi install ` | Yes | No | @@ -113,17 +113,17 @@ Run it first whenever Relay does not seem to be doing anything. | `NEMO_RELAY_PI_OPENAI_UPSTREAM` | unset | What the gateway forwards OpenAI-compatible traffic to. Set by the launcher | | `NEMO_RELAY_PI_ANTHROPIC_UPSTREAM` | unset | What the gateway forwards Anthropic traffic to. Set by the launcher | -## How tool gating works +## How Tool Gating Works For model-invoked tools, `tool_call` is the only pre-execution decision point that sees arguments — pi's `--tools`, `--exclude-tools`, `--no-tools` and runtime `setActiveTools` are all applied at tool-registry construction, never per call. The user's own inline shell takes a different path and is gated -separately; see [Inline shell](#inline-shell). +separately; see [Inline Shell](#inline-shell). The wire contract, pinned from both sides by tests: -| Gateway response | Extension behaviour | +| Gateway Response | Extension Behavior | |---|---| | 2xx | allow | | 403 with `error.type = "nemo_relay_guardrail_rejected"` | block, using `error.reason` | @@ -135,7 +135,7 @@ The block reason reaches the model **verbatim**: pi hands it to as error codes — a reason that says what to do instead produces a model that adapts rather than one that gives up. -## Argument transforms +## Argument Transforms A Relay **request intercept** can rewrite a tool's arguments. The gateway never runs the tool, so the rewrite comes back in the allow response and the extension @@ -143,11 +143,17 @@ applies it to pi's `event.input` **in place** — which is what pi documents as the mechanism, and is required: pi hands the same object to the tool and to later handlers, so replacing the reference would be discarded. -| Response body | Meaning | +| Response Body | Meaning | |---|---| | `{}` | Allow, arguments unchanged. Every non-gated hook, and any `tool_call` no intercept rewrote | | `{"tool_call": {"tool_call_id": "…", "input": {…}}}` | Allow, but execute *these* arguments | +`tool_call_id` is **echoed, and checked**: the extension applies a transform only +when the id is a string equal to the call it just posted. An envelope carrying +`input` under a missing, non-string, or different id is refused rather than +applied to whichever call happens to be open — and a refusal blocks, the same as +a shape violation does. + ⚠️ **The rewrite is constrained, not validated.** pi validates arguments *before* the `tool_call` hook and never re-validates — its own types say "no re-validation is performed after mutation" — so a rewrite that violates the @@ -183,7 +189,7 @@ the transform existed to prevent. This is a different axis from `NEMO_RELAY_PI_FAIL`, which governs an unreachable gateway rather than one that answered with something unusable. -## Inline shell +## Inline Shell pi's bang prefix runs a command without going through the tool registry: `!git status` runs it and shows the model the output, `!!git status` runs it and @@ -205,7 +211,7 @@ both**; a rule written only for `bash` does not gate the bang prefix. | `cwd` | pi's working directory for the command | | `exclude_from_context` | `true` for the `!!` form, whose output the model never sees | -### The refusal shape +### The Refusal Shape pi gives `user_bash` no block-and-reason contract — there is no `{block, reason}` here — so a refusal has to be a **synthetic failed command result**, @@ -220,16 +226,16 @@ which pi records exactly as if the command had run: `NEMO_RELAY_PI_FAIL` governs this path too: a gateway that cannot be reached allows the command by default, and refuses it under `closed` with a reason that -says explicitly that it is an infrastructure fault rather than a judgement. +says explicitly that it is an infrastructure fault rather than a judgment. **A rewritten command is refused, not run.** pi's `user_bash` result can replace the *result* or the execution backend, but never the command — both call sites pass the original text straight on to `executeBash` and read nothing back out of the event. So a request intercept that rewrites an inline command cannot be -honoured, and the command is refused rather than run unmodified, on the same +honored, and the command is refused rather than run unmodified, on the same rule the tool path applies to a transform it cannot apply safely. -### Limits worth knowing +### Limits Worth Knowing - **The gate decides; it does not observe.** pi has no completion hook for inline shell, so on an allow the span closes immediately: it measures the @@ -250,7 +256,7 @@ rule the tool path applies to a transform it cannot apply safely. component *after* the hook resolves, so a slow gateway shows nothing at all until `NEMO_RELAY_PI_TIMEOUT_MS` expires. Keep that timeout short. -## Model redirection +## Model Redirection pi resolves a base URL per model from a generated catalog, so there is no flag or environment variable to point it at the gateway. The extension does it directly: @@ -279,6 +285,7 @@ outcome as a `model_redirect` mark so a trace without LLM spans explains itself. | Gateway forwards somewhere else | Skipped, `upstream-mismatch` | | Model's API has no gateway route (Bedrock, Azure OpenAI Responses, Google, Google Vertex, Mistral, OpenAI Codex, Radius) | Skipped, `unserviceable-api` | | Launched outside `nemo-relay run --agent pi`, so the upstream is unknown | Skipped, `unknown-upstream` — set `NEMO_RELAY_PI_REDIRECT=force` to override | +| Another model of the same provider targets an endpoint the gateway does not front | Skipped, `provider-mixed-endpoints` — the registration is provider-wide, so point both upstreams at that provider | `nemo-relay run --agent pi` sets the two upstream variables for you. Running pi by hand against a standalone gateway means setting them yourself, or forcing. @@ -291,7 +298,7 @@ above do not. Count from `builtinProviders()` rather than from the 38 files in `providers/data/`: Radius is a purely dynamic provider with no static catalog entry, so a file count loses it. -## Hook mapping +## Hook Mapping pi's lifecycle is `session -> agent run -> turn -> message | tool execution`. Two shapes make a naive mapping wrong. @@ -331,7 +338,7 @@ gateway; neither is worth it here. concurrently, so `tool_execution_end` arrives out of submission order. All per-call state is keyed by `toolCallId`, the only correlator pi provides. -| pi hook | Forwarded as | Note | +| pi Hook | Forwarded As | Note | |---|---|---| | `session_start` / `session_shutdown` | session boundary | **Not** `agent_start`/`agent_end` — those repeat on re-entry. `session_shutdown` is ignored for `reason: "reload"`, which continues the same session | | `agent_start` / `agent_end` | run-level marks | Carry `attempt_index`; not a run boundary. Recorded on the session scope, not inside a turn | @@ -345,7 +352,7 @@ per-call state is keyed by `toolCallId`, the only correlator pi provides. | `tool_call` | tool start, and the gate | The only blocking hook. Carries `attempt_index`, `turn_seq` | | `tool_execution_end` | tool end | For **every** outcome, including blocked. Carries `attempt_index`, `turn_seq` | | `tool_execution_start` | *not forwarded* | Registered, but only to remember a tool name for the matching end: it fires before validation and for calls that never execute | -| `user_bash` | tool start, and the second gate | The bang prefix, which never reaches the tool registry. Gated under the tool name `user_bash` — see [Inline shell](#inline-shell) | +| `user_bash` | tool start, and the second gate | The bang prefix, which never reaches the tool registry. Gated under the tool name `user_bash` — see [Inline Shell](#inline-shell) | | *(synthesized)* `user_bash_end` | tool end | pi reports no completion for inline shell, so the extension closes the span itself | `tool_result` is deliberately unused: it does not fire for blocked calls, and in @@ -357,7 +364,7 @@ is recorded on the session scope rather than opening an empty turn to hold it. Codex and Claude Code report only `Stop`, so their turns stay lazily opened by the first event of the turn. -## What is not represented +## What Is Not Represented **Tool results are truncated at 2000 characters** before they are forwarded, with the overflow replaced by a `... [truncated N chars]` suffix. The gateway @@ -415,11 +422,11 @@ uncaught exception do not. **LLM spans, when redirection is skipped.** They are present whenever the gateway fronts the endpoint the active model would otherwise have called, and absent otherwise — the `model_redirect` mark in the trace names which it was and -why. See [Model redirection](#model-redirection). +why. See [Model Redirection](#model-redirection). **The outcome of an inline shell command.** pi reports no completion for the bang prefix, so the gate records the decision, not the command. See -[Inline shell](#inline-shell). +[Inline Shell](#inline-shell). ## Development diff --git a/integrations/pi/index.ts b/integrations/pi/index.ts index 17dabe35a..cd2accb76 100644 --- a/integrations/pi/index.ts +++ b/integrations/pi/index.ts @@ -640,7 +640,7 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { // An intercept rewrote the command. pi's result type can replace the // result or the execution backend but never the command itself, so the - // rewrite cannot be honoured -- and running the original would discard + // rewrite cannot be honored -- and running the original would discard // the policy decision. Refuse, and say which of the two it is. if (decision.kind === 'allow' && decision.body?.tool_call?.input !== undefined) { const reason = transformRefusalReason(); diff --git a/integrations/pi/src/argument-transform.ts b/integrations/pi/src/argument-transform.ts index edc334ec3..dc32667cc 100644 --- a/integrations/pi/src/argument-transform.ts +++ b/integrations/pi/src/argument-transform.ts @@ -17,7 +17,7 @@ * arguments that violate the tool's schema will execute. * 2. *The schema is reachable, and deliberately not used.* `pi.getAllTools()` * returns every configured tool -- built-ins included -- with its TypeBox - * `parameters` schema (`extensions/types.ts:1334`, impl + * `parameters` schema (pi `v0.84.0`, `core/extensions/types.ts:1334`, impl * `core/agent-session.ts:908`). So validating locally, or forwarding the * schema to the gateway, are both possible. They are not done because the * tool set is per-session mutable (`setActiveTools`, `registerTool`), so a @@ -162,6 +162,6 @@ export function refusalReason(toolName: string, detail: string): string { `A NeMo Relay policy rewrote the arguments for this ${toolName} call, but the rewrite could ` + `not be applied safely: ${detail}. The call was blocked rather than run with the original ` + `arguments, because running them would ignore the policy. This is a configuration problem in ` + - `the policy, not a judgement about your request.` + `the policy, not a judgment about your request.` ); } diff --git a/integrations/pi/src/gateway-client.ts b/integrations/pi/src/gateway-client.ts index 5dceb4246..84c6bed3d 100644 --- a/integrations/pi/src/gateway-client.ts +++ b/integrations/pi/src/gateway-client.ts @@ -144,7 +144,7 @@ export function resolveFault(config: GatewayConfig, detail: string, toolName: st reason: `The NeMo Relay policy gateway could not be reached to authorize this ${toolName} call, ` + `so it was blocked rather than allowed through unchecked. This is an infrastructure fault, ` + - `not a judgement about the request. Details: ${detail}`, + `not a judgment about the request. Details: ${detail}`, }; } diff --git a/integrations/pi/src/pi-hook-types.ts b/integrations/pi/src/pi-hook-types.ts index 06f484d5a..8c3d95e12 100644 --- a/integrations/pi/src/pi-hook-types.ts +++ b/integrations/pi/src/pi-hook-types.ts @@ -11,8 +11,23 @@ * on the pi package, which matters because pi ships breaking changes through * *minor* releases and has no major-release channel. * - * Re-verify these signatures against the pinned pi version before relying on - * them; a silent shape change would show up as missing spans, not a type error. + * Unlike OpenClaw, pi *does* publish these declarations -- `ToolCallEvent`, + * `ToolCallEventResult`, `ExtensionAPI` and `ProviderConfig` all come out of + * `@earendil-works/pi-coding-agent`'s package root. Importing them is declined on + * cost, not availability. That package is the entire agent -- TUI, provider + * stack, a wasm image codec -- and it ships an `npm-shrinkwrap.json` pinning + * ~140 further packages, every one of which would land in this repository's + * lockfile and in the Node license inventory the license-diff job walks, to type + * a file that erases to nothing at runtime. The shapes below are also + * deliberately *wider* than pi's: `toolName: string` with + * `input: Record` accepts every member of pi's per-tool + * `ToolCallEvent` union, which is what lets `src/argument-transform.ts` write + * gateway-supplied keys without narrowing per tool. + * + * The version and commit above are the contract that keeps this in step. On a pi + * bump, re-read that file and re-run `just test-pi`. Re-verify these signatures + * before relying on them; a silent shape change would show up as missing spans, + * not a type error. */ /** Fired when an agent loop starts. Carries no run identifier. */ @@ -71,7 +86,7 @@ export type SessionShutdownEvent = { * * A handler must return `undefined` here. Returning an object is how pi's API * spells "cancel this compaction, or replace its result", so an accidental - * return value from an observability hook would change pi's behaviour. + * return value from an observability hook would change pi's behavior. */ export type SessionBeforeCompactEvent = { type: 'session_before_compact'; diff --git a/integrations/pi/src/provider-redirect.ts b/integrations/pi/src/provider-redirect.ts index 6ad166241..4d7ad342f 100644 --- a/integrations/pi/src/provider-redirect.ts +++ b/integrations/pi/src/provider-redirect.ts @@ -14,9 +14,10 @@ * * With `baseUrl` and no `models`, pi rewrites the URL of every existing model * for that provider and keeps their API, headers, costs and context windows - * (`applyExtension`, `core/provider-composer.ts:215`). That is far cheaper than - * pi's own `custom-provider-*` examples, which register a `streamSimple` and - * re-implement a provider protocol; the extension stays a thin client. + * (`applyExtension`, pi `v0.84.0`, `core/provider-composer.ts:215`). That is far + * cheaper than pi's own `custom-provider-*` examples, which register a + * `streamSimple` and re-implement a provider protocol; the extension stays a + * thin client. * * **Redirection is conditional, and the condition is the whole design.** The * gateway forwards to one statically configured upstream per API family and a diff --git a/integrations/pi/src/user-bash.ts b/integrations/pi/src/user-bash.ts index 435d52512..255e8cce6 100644 --- a/integrations/pi/src/user-bash.ts +++ b/integrations/pi/src/user-bash.ts @@ -101,10 +101,10 @@ export function refusalResult(reason: string): BashResult { * pass the original string on to `executeBash`, and neither reads anything back * out of the event. Taking over execution to run the rewrite instead * would mean reimplementing pi's shell selection, command prefix and - * process-tree cancellation, which is a behaviour change the sidecar has no + * process-tree cancellation, which is a behavior change the sidecar has no * business making. * - * So the rewrite cannot be honoured, and the command is refused rather than run + * So the rewrite cannot be honored, and the command is refused rather than run * unmodified -- the same rule the tool path already applies to a transform it * cannot apply safely, and for the same reason: running the original would * silently discard a policy decision. @@ -115,6 +115,6 @@ export function transformRefusalReason(): string { 'execute a rewritten inline shell command -- the bang prefix runs the text you typed. The ' + 'command was refused rather than run unmodified, because running it would ignore the policy. ' + 'Re-run it with the change applied by hand, or ask the policy owner to gate the bash tool ' + - 'instead. This is a configuration problem in the policy, not a judgement about the command.' + 'instead. This is a configuration problem in the policy, not a judgment about the command.' ); } diff --git a/integrations/pi/test/argument-transform.test.mjs b/integrations/pi/test/argument-transform.test.mjs index c76df084b..8cb97ce1f 100644 --- a/integrations/pi/test/argument-transform.test.mjs +++ b/integrations/pi/test/argument-transform.test.mjs @@ -135,11 +135,11 @@ describe('applying the transform', () => { describe('the refusal reason', () => { // pi hands the reason to the model verbatim, so it has to read as guidance and must not look // like the model did something wrong. - it('names the tool, the cause, and that it is not a judgement of the request', () => { + it('names the tool, the cause, and that it is not a judgment of the request', () => { const reason = refusalReason('read', 'input added sudo'); assert.match(reason, /read/); assert.match(reason, /input added sudo/); - assert.match(reason, /not a judgement about your request/); + assert.match(reason, /not a judgment about your request/); assert.match(reason, /blocked rather than run with the original/); }); }); diff --git a/integrations/pi/test/gateway-client.test.mjs b/integrations/pi/test/gateway-client.test.mjs index 6a6e4e531..fd72e725d 100644 --- a/integrations/pi/test/gateway-client.test.mjs +++ b/integrations/pi/test/gateway-client.test.mjs @@ -125,7 +125,7 @@ describe('gateway client wire contract', () => { }); it('does not present a 403 without the guardrail marker as a policy decision', async () => { - // An authorization failure is not a judgement about the request; reporting + // An authorization failure is not a judgment about the request; reporting // it as one would tell the model a policy considered and refused its call. const outcome = await postHook(baseConfig(url), { hook_event_name: 'naked-403' }); assert.equal(outcome.kind, 'fault'); @@ -181,7 +181,7 @@ describe('failure policy', () => { 'read', ); assert.equal(outcome.kind, 'block'); - assert.match(outcome.reason, /infrastructure fault, not a judgement/); + assert.match(outcome.reason, /infrastructure fault, not a judgment/); assert.match(outcome.reason, /connection refused/); }); }); diff --git a/integrations/pi/test/lifecycle.test.mjs b/integrations/pi/test/lifecycle.test.mjs index 8ce37eeb2..4b7aa3564 100644 --- a/integrations/pi/test/lifecycle.test.mjs +++ b/integrations/pi/test/lifecycle.test.mjs @@ -8,7 +8,7 @@ * exercised the handlers themselves, so the identity fields the gateway cannot * infer -- `attempt_index` and `turn_seq` -- were implemented and demonstrated * once in a live trace but never pinned. These tests pin them, plus the - * shutdown-reason behaviour. + * shutdown-reason behavior. * * Run: node --test integrations/pi/test/*.test.mjs */ @@ -316,7 +316,7 @@ describe('compaction', () => { }); // pi spells "cancel this compaction, or replace its result" as a returned object, so an - // observability handler that returned anything would change pi's behaviour. + // observability handler that returned anything would change pi's behavior. it('never returns a value that could cancel or replace pi compaction', async () => { const fire = load(); await fire('session_start', { reason: 'startup' }); diff --git a/integrations/pi/test/tool-call.test.mjs b/integrations/pi/test/tool-call.test.mjs index 04f5519f7..6210b39a2 100644 --- a/integrations/pi/test/tool-call.test.mjs +++ b/integrations/pi/test/tool-call.test.mjs @@ -109,7 +109,7 @@ describe('the tool_call gate', () => { assert.match(result.reason, /added sudo/); // Falling back to the original arguments would silently discard a policy // decision, which is the failure the transform exists to prevent. - assert.match(result.reason, /not a judgement about your request/); + assert.match(result.reason, /not a judgment about your request/); assert.deepEqual(event.input, { path: '/work/.env' }, 'a refused rewrite must not be applied'); }); @@ -138,7 +138,7 @@ describe('the tool_call gate', () => { assert.equal(result?.block, true); // Telling the model a policy considered and refused its call, when nothing // did, gives it a false premise to reason from. - assert.match(result.reason, /infrastructure fault, not a judgement/); + assert.match(result.reason, /infrastructure fault, not a judgment/); }); it('treats a 403 without the guardrail marker as a fault, not a policy decision', async () => { @@ -181,7 +181,7 @@ describe('an extension ahead of the gate', () => { beforeEach(() => gateway.reset()); - // The documented blind spot, pinned as behaviour rather than prose: pi stops + // The documented blind spot, pinned as behavior rather than prose: pi stops // at the first handler that returns anything, so on the `pi install` path -- // which loads last -- the call is blocked and the gateway never learns it // happened. `-e` inverts the order, which is why the launcher uses it. diff --git a/integrations/pi/test/user-bash.test.mjs b/integrations/pi/test/user-bash.test.mjs index bd6757f1b..923a454c2 100644 --- a/integrations/pi/test/user-bash.test.mjs +++ b/integrations/pi/test/user-bash.test.mjs @@ -118,7 +118,7 @@ describe('inline shell gate', () => { const fire = load(); // An allow, but with rewritten arguments. pi's user_bash result type can // replace the result or the execution backend, never the command, so the - // rewrite cannot be honoured. + // rewrite cannot be honored. gateway.replyWith({ status: 200, payload: { tool_call: { tool_call_id: 'user-bash-0', input: { command: 'git status --short' } } }, @@ -150,7 +150,7 @@ describe('inline shell gate', () => { } }); - it('fails closed on demand, and says it is an infrastructure fault rather than a judgement', async () => { + it('fails closed on demand, and says it is an infrastructure fault rather than a judgment', async () => { process.env.NEMO_RELAY_PI_FAIL = 'closed'; process.env.NEMO_RELAY_PI_GATEWAY_URL = 'http://127.0.0.1:1'; const fire = load(); @@ -164,7 +164,7 @@ describe('inline shell gate', () => { assert.equal(result.result.exitCode, REFUSED_EXIT_CODE); // Telling the user a policy considered and refused their command, when // nothing did, gives them a false premise to act on. - assert.match(result.result.output, /infrastructure fault, not a judgement/); + assert.match(result.result.output, /infrastructure fault, not a judgment/); } finally { process.env.NEMO_RELAY_PI_GATEWAY_URL = url; } @@ -185,7 +185,7 @@ describe('inline shell gate', () => { }); assert.ok(result?.result, 'an unreadable success must not fall through to an allow'); assert.equal(result.result.exitCode, REFUSED_EXIT_CODE); - assert.match(result.result.output, /infrastructure fault, not a judgement/); + assert.match(result.result.output, /infrastructure fault, not a judgment/); }); it('forwards the !! form so a policy can see the output will bypass the model', async () => { From 17ee262c3b9837699baa204d8a396f8fe3440773 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 09:58:10 -0700 Subject: [PATCH 27/41] test(pi): assert the guardrail saw the pre-rewrite arguments, not just once A counter alone proves one evaluation happened, not *which* arguments it decided on -- move the conditional check after the intercept and the test still passes with a count of one. It now records every argument object the guardrail was handed and asserts the whole sequence, which catches both failure modes: a second pass, and a single pass on the rewrite. Recorded rather than asserted inside the closure on purpose. The runtime runs guardrail callbacks under `catch_unwind`, so a panic there becomes `FlowError::Internal` and reaches the test as a 500 -- indistinguishable from a guardrail that genuinely errored, and with the actual mismatch nowhere in the output. Verified by moving the check: the assertion fails with the two argument objects side by side. Also: "15 lifecycle hooks" was an overclaim in both places it appeared. Three of the fifteen are synthesized by the extension rather than emitted by pi, and the set spans tool and inline-shell activity as well as lifecycle. Both surfaces now say what the fifteen are, and the `user_bash_end` row is keyed like the other two synthesized events rather than posing as a pi hook. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- .../cli/tests/coverage/shared/server_tests.rs | 19 +++++++++++++------ docs/nemo-relay-cli/pi.mdx | 15 ++++++++++----- docs/reference/support-matrix.mdx | 2 +- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/crates/cli/tests/coverage/shared/server_tests.rs b/crates/cli/tests/coverage/shared/server_tests.rs index cabe70878..498bc2d74 100644 --- a/crates/cli/tests/coverage/shared/server_tests.rs +++ b/crates/cli/tests/coverage/shared/server_tests.rs @@ -4729,16 +4729,20 @@ async fn pi_tool_call_hook_returns_arguments_a_request_intercept_rewrote() { #[tokio::test] async fn pi_tool_call_hook_evaluates_conditional_guardrails_once_when_an_intercept_rewrites() { let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; - static EVALUATIONS: AtomicUsize = AtomicUsize::new(0); - EVALUATIONS.store(0, Ordering::SeqCst); + // Recorded rather than asserted inside the closure: the runtime runs guardrail callbacks + // under `catch_unwind`, so a panic there becomes `FlowError::Internal` and would surface as + // a 500 -- indistinguishable from a guardrail that genuinely errored. + let seen: Arc>> = Arc::new(std::sync::Mutex::new(Vec::new())); + let recorder = Arc::clone(&seen); let _ = deregister_tool_conditional_execution_guardrail("cli-pi-transform-counter"); register_tool_conditional_execution_guardrail( "cli-pi-transform-counter", 1, - Arc::new(|_name, _args| { + Arc::new(move |_name, args| { + let recorder = Arc::clone(&recorder); Box::pin(async move { - EVALUATIONS.fetch_add(1, Ordering::SeqCst); + recorder.lock().unwrap().push(args); Ok(None) }) }), @@ -4790,9 +4794,12 @@ async fn pi_tool_call_hook_evaluates_conditional_guardrails_once_when_an_interce let bytes = response.into_body().collect().await.unwrap().to_bytes(); let body: Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(body["tool_call"]["input"], json!({ "path": "/work/.env" })); + // One evaluation, and on the pre-rewrite arguments. Asserting the whole sequence catches both + // failure modes a count alone cannot: a second pass, and a single pass moved after the + // intercept, which would record `/work/.env` here and still count one. assert_eq!( - EVALUATIONS.load(Ordering::SeqCst), - 1, + *seen.lock().unwrap(), + vec![json!({ "path": "/work/README.md" })], "the conditional chain must decide once, on the arguments pi proposed" ); } diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index d54a80b02..ab33b313d 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -265,10 +265,15 @@ type a bang prefix into. ## Captured Events -The extension posts 15 hooks. pi reports both ends of a turn, so Relay opens the -turn scope at pi's own `turn_start` instead of inferring one, and a mark that -arrives between turns is recorded on the session scope rather than opening an -empty turn to hold it. +The extension posts 15 event types, covering session, turn, tool and +inline-shell activity. Twelve carry a pi hook's own name. The other three — +`model_redirect`, `tool_arguments_transformed` and `user_bash_end` — are +synthesized by the extension, and the table below shows what each is derived +from rather than a hook pi emits. + +pi reports both ends of a turn, so Relay opens the turn scope at pi's own +`turn_start` instead of inferring one, and a mark that arrives between turns is +recorded on the session scope rather than opening an empty turn to hold it. | pi hook | Relay lifecycle | | --- | --- | @@ -282,7 +287,7 @@ empty turn to hold it. | `tool_call` | Tool span start, and the gate | | `tool_execution_end` | Tool span end, for every outcome including blocked | | `user_bash` | Tool span start named `user_bash`, and the inline shell gate | -| `user_bash_end` | Tool span end. Synthesized by the extension, because pi reports no completion for inline shell | +| *(after `user_bash` completes)* | `user_bash_end`, a tool span end. Synthesized by the extension, because pi reports no completion for inline shell | pi's `tool_execution_start` is deliberately not forwarded: it fires before argument validation and also for calls that never execute. `tool_result` is not diff --git a/docs/reference/support-matrix.mdx b/docs/reference/support-matrix.mdx index 608e2fac4..a72f85a05 100644 --- a/docs/reference/support-matrix.mdx +++ b/docs/reference/support-matrix.mdx @@ -66,7 +66,7 @@ and older CLI versions during installation, diagnostics, and transparent runs. | --- | --- | --- | --- | | Claude Code | 2.1.121 | Persistent install, transparent run, lifecycle hooks, local gateway routing, and pre-tool security | Claude desktop, web, and application sessions are unsupported unless they expose the same local hook and gateway controls. Optimization requires gateway-routed LLM traffic and available hooks. | | Codex CLI | 0.143.0 | Persistent install, transparent run, 10 supported plugin hooks, local gateway routing, and pre-tool security | Cloud or remote tasks that bypass the local machine have partial or no LLM capture. The plugin hook schema has no `SessionEnd`; Relay finalizes the cumulative session snapshot at `Stop`. Encrypted Codex multi-agent v2 payloads cannot be decrypted or reliably linked. | -| pi | 0.84.0 | Transparent run through a Relay-authored pi extension, 15 lifecycle hooks, tool-call security, inline-shell security, and model-call security when redirection applies | Proof of concept. No persistent install: pi has no plugin marketplace. Model traffic is redirected only when the gateway forwards to the endpoint every model of the selected provider would otherwise call, because pi's provider registration is provider-wide; otherwise there are no LLM spans for that model. Seven of pi's 39 built-in providers speak an API the gateway has no route for. Subagents and nested pi processes appear as unrelated sessions. pi ships breaking changes through minor releases, so hook signatures need re-verification after an upgrade. | +| pi | 0.84.0 | Transparent run through a Relay-authored pi extension, 15 forwarded event types covering session, turn, tool and inline-shell activity, tool-call security, inline-shell security, and model-call security when redirection applies | Proof of concept. No persistent install: pi has no plugin marketplace. Model traffic is redirected only when the gateway forwards to the endpoint every model of the selected provider would otherwise call, because pi's provider registration is provider-wide; otherwise there are no LLM spans for that model. Seven of pi's 39 built-in providers speak an API the gateway has no route for. Subagents and nested pi processes appear as unrelated sessions. pi ships breaking changes through minor releases, so hook signatures need re-verification after an upgrade. | For installation, diagnostics, and host-specific behavior, refer to [Coding Agent Installation](/nemo-relay-cli/plugin-installation), [Claude From 27cf84d30637c4e1e8f10819104a867daeba350a Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 10:10:43 -0700 Subject: [PATCH 28/41] docs(pi): name where each synthesized event actually comes from The previous wording said the table shows what the three synthesized events derive from "rather than a hook pi emits", while the table keyed `model_redirect` on `model_select` -- a pi hook. The review that caught it proposed saying `model_redirect` derives from `model_select`, which is half true: it is posted from `session_start` as well, which is the first decision of the session and the one a user with no LLM spans is looking for. Both tables now key it on both sources and mark it, and `tool_arguments_transformed`, as synthesized. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- docs/nemo-relay-cli/pi.mdx | 10 ++++++---- integrations/pi/README.md | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index ab33b313d..4999a4a61 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -268,8 +268,10 @@ type a bang prefix into. The extension posts 15 event types, covering session, turn, tool and inline-shell activity. Twelve carry a pi hook's own name. The other three — `model_redirect`, `tool_arguments_transformed` and `user_bash_end` — are -synthesized by the extension, and the table below shows what each is derived -from rather than a hook pi emits. +synthesized by the extension. The table below names the source of each: +`model_redirect` is posted from `session_start` and again from every +`model_select`, `tool_arguments_transformed` after a rewrite is applied, and +`user_bash_end` after an inline-shell command is decided. pi reports both ends of a turn, so Relay opens the turn scope at pi's own `turn_start` instead of inferring one, and a mark that arrives between turns is @@ -282,8 +284,8 @@ recorded on the session scope rather than opening an empty turn to hold it. | `turn_start` / `turn_end` | Turn scope open and close | | `session_before_compact` | Mark. The compaction is announced, not yet done, and a later extension can still cancel it | | `session_compact` | Canonical `compaction` mark | -| `model_select` | `model_redirect` mark recording whether redirection applied | -| *(after a rewrite)* | `tool_arguments_transformed` mark, so the trace records that the arguments the tool ran were not the ones proposed | +| `session_start`, then every `model_select` | `model_redirect` mark recording whether redirection applied. Synthesized: the decision is re-evaluated per model, so a switch away from a provider the gateway fronts stops redirecting | +| *(after a rewrite)* | `tool_arguments_transformed` mark. Synthesized, so the trace records that the arguments the tool ran were not the ones proposed | | `tool_call` | Tool span start, and the gate | | `tool_execution_end` | Tool span end, for every outcome including blocked | | `user_bash` | Tool span start named `user_bash`, and the inline shell gate | diff --git a/integrations/pi/README.md b/integrations/pi/README.md index 647da4194..0037fe323 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -345,7 +345,7 @@ per-call state is keyed by `toolCallId`, the only correlator pi provides. | `agent_settled` | run-level mark | Fires exactly once, from a `finally`. Carries `attempts` (the count) and `attempt_index` (the last one) | | `turn_start` | turn scope **open** | Carries `turn_index`, `turn_seq`, `attempt_index`. Awaited, so the turn exists before pi's model call arrives | | `turn_end` | turn scope **close** | Carries `turn_index`, `turn_seq`, `attempt_index`. Awaited, for the same reason | -| `model_select` | `model_redirect` mark | Re-evaluates redirection for the newly selected model | +| `session_start`, then every `model_select` | `model_redirect` mark | Synthesized. Re-evaluates redirection for the newly selected model, so a switch away from a provider the gateway fronts stops redirecting | | *(after a rewrite)* | `tool_arguments_transformed` mark | Synthesized, so the trace records that the arguments the tool ran were not the ones proposed | | `session_before_compact` | mark | Announced, not done, and cancellable by a later extension. Carries `reason`, `will_retry`, `tokens_before` | | `session_compact` | compaction | The completed compaction, which the runtime treats as proof the context was rebuilt | From bda027f824f783a65bafaac8d0990f11441b79b3 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 10:39:14 -0700 Subject: [PATCH 29/41] docs(pi): evaluation is per model select; the mark is not `isNotable` gates emission: a decision of `no-model` or `already-redirected` is evaluated and then dropped, because a mark per `session_start` saying "no model yet" is noise in every trace. Four places said or implied otherwise -- "every outcome is recorded", and two table rows keyed on `session_start`, then every `model_select` with no mention that the posting is conditional. They now separate the two: redirection is evaluated on each selection, and `model_redirect` is posted for each decision that explains something. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- docs/nemo-relay-cli/pi.mdx | 17 +++++++++++------ integrations/pi/README.md | 9 ++++++--- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index 4999a4a61..e0456b4e8 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -269,8 +269,11 @@ The extension posts 15 event types, covering session, turn, tool and inline-shell activity. Twelve carry a pi hook's own name. The other three — `model_redirect`, `tool_arguments_transformed` and `user_bash_end` — are synthesized by the extension. The table below names the source of each: -`model_redirect` is posted from `session_start` and again from every -`model_select`, `tool_arguments_transformed` after a rewrite is applied, and +redirection is evaluated at `session_start` and again on every `model_select`, +and `model_redirect` is posted for each decision that explains something — a +redirect, or a skip an operator can act on. Two transient skips are evaluated but +not posted: no model resolved yet, and a provider already pointed at the gateway. +`tool_arguments_transformed` is posted after a rewrite is applied, and `user_bash_end` after an inline-shell command is decided. pi reports both ends of a turn, so Relay opens the turn scope at pi's own @@ -284,7 +287,7 @@ recorded on the session scope rather than opening an empty turn to hold it. | `turn_start` / `turn_end` | Turn scope open and close | | `session_before_compact` | Mark. The compaction is announced, not yet done, and a later extension can still cancel it | | `session_compact` | Canonical `compaction` mark | -| `session_start`, then every `model_select` | `model_redirect` mark recording whether redirection applied. Synthesized: the decision is re-evaluated per model, so a switch away from a provider the gateway fronts stops redirecting | +| `session_start`, then every `model_select` | `model_redirect` mark, for each decision that explains something. Synthesized: the decision is re-evaluated per model, so a switch away from a provider the gateway fronts stops redirecting | | *(after a rewrite)* | `tool_arguments_transformed` mark. Synthesized, so the trace records that the arguments the tool ran were not the ones proposed | | `tool_call` | Tool span start, and the gate | | `tool_execution_end` | Tool span end, for every outcome including blocked | @@ -389,9 +392,11 @@ has never heard of them. Pointing both `--openai-base-url` and `--anthropic-base-url` at that provider unblocks it, unless a sibling speaks an API the gateway has no route for at all. -Every outcome is recorded as a `model_redirect` mark on the session scope, so a -trace with no LLM spans states its own reason. The decision is re-made on each -model switch. +Each of these outcomes is recorded as a `model_redirect` mark on the session +scope, so a trace with no LLM spans states its own reason. The decision is +re-made on each model switch; the two skips that explain nothing — no model +resolved yet, and a provider already redirected — are evaluated but not marked, +because one per `session_start` is noise in every trace. To capture a specific provider, point the gateway at it: diff --git a/integrations/pi/README.md b/integrations/pi/README.md index 0037fe323..dcb00941a 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -276,8 +276,11 @@ one statically configured upstream per API family — `--openai-base-url` and redirecting is only correct when the gateway's upstream *is* the endpoint the selected model would otherwise call. Pointing an NVIDIA model at a gateway configured for `api.openai.com` does not degrade to "no spans"; it breaks the -session. The extension therefore redirects only on a match, and records every -outcome as a `model_redirect` mark so a trace without LLM spans explains itself. +session. The extension therefore redirects only on a match, and records each +outcome that explains something as a `model_redirect` mark so a trace without LLM +spans accounts for itself. Two transient skips are evaluated but not marked — no +model resolved yet, and a provider already pointed at the gateway — because a +mark per `session_start` for either is noise. | Situation | Outcome | |---|---| @@ -345,7 +348,7 @@ per-call state is keyed by `toolCallId`, the only correlator pi provides. | `agent_settled` | run-level mark | Fires exactly once, from a `finally`. Carries `attempts` (the count) and `attempt_index` (the last one) | | `turn_start` | turn scope **open** | Carries `turn_index`, `turn_seq`, `attempt_index`. Awaited, so the turn exists before pi's model call arrives | | `turn_end` | turn scope **close** | Carries `turn_index`, `turn_seq`, `attempt_index`. Awaited, for the same reason | -| `session_start`, then every `model_select` | `model_redirect` mark | Synthesized. Re-evaluates redirection for the newly selected model, so a switch away from a provider the gateway fronts stops redirecting | +| `session_start`, then every `model_select` | `model_redirect` mark, for each decision that explains something | Synthesized. Re-evaluates redirection for the newly selected model, so a switch away from a provider the gateway fronts stops redirecting | | *(after a rewrite)* | `tool_arguments_transformed` mark | Synthesized, so the trace records that the arguments the tool ran were not the ones proposed | | `session_before_compact` | mark | Announced, not done, and cancellable by a later extension. Carries `reason`, `will_retry`, `tokens_before` | | `session_compact` | compaction | The completed compaction, which the runtime treats as proof the context was rebuilt | From 3a9fa202d000ca5f3d7dc048cca9b0fdddeb70d6 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 11:19:56 -0700 Subject: [PATCH 30/41] fix(pi): two copies load, and an installed one could not be seen Two holes in extension resolution that a review found, both verified against pi v0.84.0 rather than reasoned about. **`-e` adds to pi's extension set; it does not replace it.** My own comment claimed the load was safe because pi de-duplicates the merged command-line and discovered sets -- true, but only by *canonicalized path*. Two distinct checkouts of one package are two identities to pi (`getPackageIdentity` gives a local source `local:`), so an explicit `NEMO_RELAY_PI_EXTENSION` pointing at checkout A while checkout B is installed loads both: two factory calls, two handler maps, every hook posted twice. A duplicated `turn_start` closes the turn its twin just opened as superseded, and the inline-shell gate decides one command twice with the second verdict the one the user gets. (The model-tool gate is unaffected -- `start_tool` returns early on a known call id.) The launcher now refuses and names both copies, and `doctor` reports the same condition, which is reachable with no Relay command involved at all: pi scans its extensions directory and its recorded packages independently. An in-extension guard was considered and rejected. It would have to tell a sibling copy from a runtime pi has since torn down, and the only signals for that are pi internals -- get it wrong and the extension registers nothing after `/reload`, a silent total loss of governance, strictly worse than the duplicate. **A `packages` entry can be an object, and only strings were read.** pi accepts `string | {source, autoload?, extensions?, ...}` and resolves both through one path. This was not a hand-edit shape: pi's own configuration selector rewrites a string entry into the object form the moment a user toggles any resource of that package, so one keystroke in pi's UI made `doctor` report an installed extension as missing and made `run --agent pi` refuse to start. Both forms are read now, and the two filter shapes that leave a package's extensions disabled -- an empty `extensions` array, and `autoload: false` with no patterns -- are reported as such rather than as absent or as a plain Pass. The launch path deliberately ignores that flag: `-e` applies no settings filter, so the launcher still instruments a session the user's own `pi` runs are missing. Three pieces of user-facing text were left contradicting the code by the previous round, and are corrected here because they are its consequences: - The trust warning told a project-scoped user to run `nemo-relay run --agent pi` instead -- the one thing that now refuses a project-scoped copy. It names the two routes the launcher does resolve. - `nemo-relay launch pi` is not a command and never was. It appeared in the marketplace-unsupported error, which a dozen call sites return, and in the extension's own header. - The README called `NEMO_RELAY_PI_EXTENSION` an override. It is the highest-precedence *candidate*: ignored unless the path exists and its manifest names this package, after which resolution falls through. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- crates/cli/src/agents/mod.rs | 2 +- crates/cli/src/agents/pi/doctor.rs | 164 ++++++++++++++--- crates/cli/src/agents/pi/launch.rs | 28 ++- crates/cli/src/diagnostics/mod.rs | 37 +++- crates/cli/src/sessions/mod.rs | 5 + .../tests/coverage/agents/launcher_tests.rs | 43 +++++ .../tests/coverage/agents/pi_doctor_tests.rs | 165 ++++++++++++++++++ .../cli/tests/coverage/shared/server_tests.rs | 70 ++++++++ docs/nemo-relay-cli/pi.mdx | 8 +- integrations/pi/README.md | 2 +- integrations/pi/index.ts | 2 +- 11 files changed, 492 insertions(+), 34 deletions(-) diff --git a/crates/cli/src/agents/mod.rs b/crates/cli/src/agents/mod.rs index 9b03f024d..2c8f0a3e0 100644 --- a/crates/cli/src/agents/mod.rs +++ b/crates/cli/src/agents/mod.rs @@ -43,7 +43,7 @@ pub(super) struct AgentDescriptor { /// rejects pi explicitly and the gateway surface (hooks, launch, doctor) is /// implemented for real. pub(crate) const PI_MARKETPLACE_UNSUPPORTED: &str = "pi has no plugin marketplace; install the NeMo Relay pi extension with `pi install ` \ - or place it in `~/.pi/agent/extensions/`, then run pi through `nemo-relay launch pi`"; + or place it in `~/.pi/agent/extensions/`, then run pi through `nemo-relay run --agent pi`"; /// Reached only if a marketplace code path forgets to reject pi first. macro_rules! pi_marketplace_unreachable { diff --git a/crates/cli/src/agents/pi/doctor.rs b/crates/cli/src/agents/pi/doctor.rs index 12ef2b257..1a4ab809e 100644 --- a/crates/cli/src/agents/pi/doctor.rs +++ b/crates/cli/src/agents/pi/doctor.rs @@ -72,6 +72,18 @@ impl ExtensionScope { pub(crate) struct ExtensionSite { pub(crate) path: PathBuf, pub(crate) scope: ExtensionScope, + /// Whether pi's own settings switch this copy off. + /// + /// An object-form `packages` entry carries per-resource filters, and two shapes leave nothing + /// enabled for extensions: an empty `extensions` array, and `autoload: false` with no + /// extension patterns (pi `v0.84.0`, `core/package-manager.ts:2208` and `:2232`). The copy is + /// installed and pi still never loads it, which is the same silent drop as the trust gate and + /// must not read as a plain Pass. + /// + /// A launch is unaffected on purpose: `-e` resolves its argument with no filter at all, so + /// `launchable_extension_path` ignores this flag. The user asked for instrumentation by + /// running the launcher; the warning is for their own `pi` sessions. + pub(crate) disabled_by_settings: bool, } /// Human-readable hook status for `nemo-relay doctor`. @@ -164,6 +176,7 @@ pub(crate) fn relay_extension_sites(cwd: &Path) -> Vec { sites.push(ExtensionSite { path, scope: ExtensionScope::Explicit, + disabled_by_settings: false, }); } if let Some(dir) = user_extensions_dir() @@ -172,26 +185,32 @@ pub(crate) fn relay_extension_sites(cwd: &Path) -> Vec { sites.push(ExtensionSite { path, scope: ExtensionScope::User, + disabled_by_settings: false, }); } if let Some(settings) = user_settings_path() - && let Some(path) = relay_package_in_settings(&settings) + && let Some(install) = relay_package_in_settings(&settings) { sites.push(ExtensionSite { - path, + path: install.path, scope: ExtensionScope::User, + disabled_by_settings: install.disabled, }); } if let Some(path) = relay_entry_in_directory(&cwd.join(PI_CONFIG_DIR).join("extensions")) { sites.push(ExtensionSite { path, scope: ExtensionScope::Project, + disabled_by_settings: false, }); } - if let Some(path) = relay_package_in_settings(&cwd.join(PI_CONFIG_DIR).join(PI_SETTINGS_FILE)) { + if let Some(install) = + relay_package_in_settings(&cwd.join(PI_CONFIG_DIR).join(PI_SETTINGS_FILE)) + { sites.push(ExtensionSite { - path, + path: install.path, scope: ExtensionScope::Project, + disabled_by_settings: install.disabled, }); } sites @@ -214,7 +233,9 @@ pub(crate) fn relay_extension_sites(cwd: &Path) -> Vec { /// resolve a package directory through the same `pi.extensions` manifest, so the /// extension loads -- and registers its hooks -- exactly once. Passing the /// directory is also why nothing here reads that manifest: pi does it, and its -/// entry-point precedence is pi's to change. +/// entry-point precedence is pi's to change. A copy pi would *not* have +/// discovered is a different matter, and is what `conflicting_extension_site` +/// is for. pub(crate) fn launchable_extension_path(cwd: &Path) -> Option { relay_extension_sites(cwd) .into_iter() @@ -222,6 +243,52 @@ pub(crate) fn launchable_extension_path(cwd: &Path) -> Option { .map(|site| site.path) } +/// A *second* copy of this extension that pi would load beside the launched one. +/// +/// `-e` adds to pi's extension set; it does not replace it. pi merges the +/// command-line and discovered sets and de-duplicates them by canonicalized path +/// alone (`mergePaths`, pi `v0.84.0`, `core/resource-loader.ts:845`), and the +/// identity it gives a local package is that same path (`getPackageIdentity`, +/// `core/package-manager.ts:1660`) -- so **nothing in pi notices that two +/// directories hold one package**. Each copy gets its own factory call and its own +/// handler map (`core/extensions/loader.ts:506`), and the runner walks every +/// extension for every hook (`core/extensions/runner.ts:805`), so every hook is +/// posted twice. A duplicated `turn_start` closes the turn its twin just opened as +/// superseded, and the inline-shell gate decides one command twice under two +/// spans, with the second verdict the one the user gets. +/// +/// Compared by *package root* rather than by path, because one install is +/// reachable both as its directory and as the entry file inside it, and pi +/// resolves both to the same file through the `pi.extensions` manifest. Symlinks +/// are resolved because pi resolves them too -- its `canonicalizePath` is +/// `realpathSync` (`utils/paths.ts:28`) -- so a symlinked copy is one copy to pi +/// and must be one copy here. +/// +/// Project scope is excluded: pi loads a project-scoped extension only for a +/// trusted project, so it is not reliably a second load, and refusing on it would +/// block launches that are fine. The existing trust warning already names it. +pub(crate) fn conflicting_extension_site(cwd: &Path, launched: &Path) -> Option { + let launched_root = package_root(launched); + relay_extension_sites(cwd) + .into_iter() + .filter(|site| site.scope != ExtensionScope::Project) + .find(|site| package_root(&site.path) != launched_root) + .map(|site| site.path) +} + +/// The package directory a site belongs to, or `None` when it is not on disk. +/// +/// A source that is not a path -- an npm or git specifier `pi install` recorded -- +/// has no root to compare and is a separate installed copy by construction, so +/// `None` is the honest answer and makes it compare unequal to a real checkout. +fn package_root(path: &Path) -> Option { + let canonical = std::fs::canonicalize(path).ok()?; + if manifest_names_relay(&canonical.join("package.json")) { + return Some(canonical); + } + canonical.parent().map(Path::to_path_buf) +} + /// The NeMo Relay extension inside a pi auto-discovery directory, if it is there. /// /// Matched on the package name, not on "the directory is non-empty". A user with @@ -260,31 +327,82 @@ fn manifest_names_relay(manifest: &Path) -> bool { .unwrap_or(false) } +/// A `packages` entry that records this extension, and whether pi will load it. +struct RecordedInstall { + path: PathBuf, + disabled: bool, +} + /// The NeMo Relay package among the sources `pi install` recorded, if any. /// -/// Each entry is a source string, and a local one is a path relative to the -/// settings file's own directory. Only a local source can be resolved from here -/// -- an npm or git source is a name, not a location -- so those fall back to +/// Each entry is **either a source string or an object** carrying that same source +/// under `source` alongside per-resource filters (pi `v0.84.0`, +/// `core/settings-manager.ts:72-87`); both shapes resolve through one code path in +/// pi. Reading only the string shape was not a theoretical gap: pi's own +/// configuration selector rewrites a string entry into the object form the moment a +/// user toggles any resource of that package +/// (`interactive/components/config-selector.ts:595-598`), so one keystroke in pi's +/// own UI made this check report an installed extension as missing -- and the +/// launcher, which shares this resolution, refuse to start. +/// +/// A local source is a path relative to the settings file's own directory, which is +/// where pi resolves it from too (`getBaseDirForScope`, +/// `core/package-manager.ts:2107-2115`). Only a local source can be resolved from +/// here -- an npm or git source is a name, not a location -- so those fall back to /// matching the package name inside the specifier, which is the best signal /// available without fetching anything. -fn relay_package_in_settings(settings: &Path) -> Option { +fn relay_package_in_settings(settings: &Path) -> Option { let base = settings.parent()?; let raw = std::fs::read_to_string(settings).ok()?; let value: serde_json::Value = serde_json::from_str(&raw).ok()?; - value - .get("packages")? - .as_array()? - .iter() - .filter_map(serde_json::Value::as_str) - .find_map(|source| { - let resolved = base.join(source); - if is_relay_extension(&resolved) { - return Some(resolved); - } - source - .contains(RELAY_PACKAGE_NAME) - .then(|| PathBuf::from(source)) - }) + value.get("packages")?.as_array()?.iter().find_map(|entry| { + let source = package_source(entry)?; + let disabled = entry_disables_extensions(entry); + let resolved = base.join(source); + if is_relay_extension(&resolved) { + return Some(RecordedInstall { + path: resolved, + disabled, + }); + } + source + .contains(RELAY_PACKAGE_NAME) + .then(|| RecordedInstall { + path: PathBuf::from(source), + disabled, + }) + }) +} + +/// The source string of one `packages` entry, whichever shape it was written in. +fn package_source(entry: &serde_json::Value) -> Option<&str> { + entry + .as_str() + .or_else(|| entry.get("source").and_then(serde_json::Value::as_str)) +} + +/// Whether an object-form entry's filters switch that package's extensions off. +/// +/// Only the two shapes pi decides without consulting the package manifest are +/// recognized: an empty `extensions` array disables every extension file in the +/// package (`applyPackageFilter`, pi `v0.84.0`, `core/package-manager.ts:2208`), +/// and `autoload: false` starts from nothing, so an entry adding no `extensions` +/// patterns adds nothing back (`applyPackageDeltaFilter`, `:2232`). +/// +/// A non-empty pattern list is matched against the manifest, which this module does +/// not read, so it counts as enabled. Guessing wrong in that direction produces the +/// false negative this module exists to prevent. +fn entry_disables_extensions(entry: &serde_json::Value) -> bool { + let Some(object) = entry.as_object() else { + return false; + }; + match object + .get("extensions") + .and_then(serde_json::Value::as_array) + { + Some(patterns) => patterns.is_empty(), + None => object.get("autoload") == Some(&serde_json::Value::Bool(false)), + } } /// `/settings.json`, where `pi install` records a user-scope package. diff --git a/crates/cli/src/agents/pi/launch.rs b/crates/cli/src/agents/pi/launch.rs index 63e6470ad..8ce0e682d 100644 --- a/crates/cli/src/agents/pi/launch.rs +++ b/crates/cli/src/agents/pi/launch.rs @@ -70,6 +70,22 @@ pub(crate) fn prepare( code pi itself would not trust. `nemo-relay doctor pi` reports what was found" ))); }; + // `-e` loads *in addition to* discovery, so a second copy of this extension elsewhere on the + // machine is a second package to pi and loads beside this one, posting every hook twice: each + // turn closed as superseded by its own duplicate, and the inline-shell gate deciding one + // command twice. There is no way to suppress it from here that does not also drop the user's + // own extensions -- `--no-extensions` keeps `-e` but discards everything discovered -- so + // refuse and name both copies rather than launch a session whose trace is doubled. + if let Some(duplicate) = conflicting_extension_path(&path) { + return Err(CliError::Launch(format!( + "two copies of the NeMo Relay pi extension would load in the same session: {} and \ + {}. pi de-duplicates its extension set by path, not by package, so both register \ + hooks and every turn, tool and inline-shell event is reported twice. Remove one \ + copy, or point {PI_EXTENSION_PATH_ENV} at the one you keep", + path.display(), + duplicate.display() + ))); + } let rendered = path.display().to_string(); set_env(launch, PI_EXTENSION_PATH_ENV, &rendered); insert_after_host( @@ -107,6 +123,14 @@ fn set_env(launch: &mut PreparedAgentLaunch, name: &str, value: &str) { /// to set. `launchable_extension_path` also excludes the project scope, which /// `-e` would load past pi's trust gate. fn extension_path() -> Option { - let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); - super::doctor::launchable_extension_path(&cwd) + super::doctor::launchable_extension_path(¤t_dir()) +} + +/// The other copy pi would load beside the launched one, if there is one. +fn conflicting_extension_path(launched: &std::path::Path) -> Option { + super::doctor::conflicting_extension_site(¤t_dir(), launched) +} + +fn current_dir() -> PathBuf { + std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")) } diff --git a/crates/cli/src/diagnostics/mod.rs b/crates/cli/src/diagnostics/mod.rs index 35783d499..de1116308 100644 --- a/crates/cli/src/diagnostics/mod.rs +++ b/crates/cli/src/diagnostics/mod.rs @@ -534,14 +534,45 @@ fn pi_extension_trust_check(cwd: &Path) -> Check { "{} is project-scoped, so pi loads it only when the project is trusted, and \ `-p`, `--mode json` and `--mode rpc` never prompt -- it is silently skipped \ there, with nothing reporting it. Install at user scope \ - (`~/.pi/agent/extensions/`, or `pi install` without `--local`), or launch with \ - `nemo-relay run --agent pi`, which passes `-e` and is never trust-gated", - project.path.display() + (`~/.pi/agent/extensions/`, or `pi install` without `--local`), or set {} to \ + it. `nemo-relay run --agent pi` passes `-e`, which is never trust-gated, but it \ + resolves only those two routes and refuses a project-scoped copy for the same \ + reason this warns about one", + project.path.display(), + crate::agents::pi::launch::PI_EXTENSION_PATH_ENV ), }; } match sites.first() { + // Reachable without the launcher at all: pi scans its extensions directory and its + // recorded packages independently, so a user holding both a copy and an install + // double-loads under plain `pi`, with no Relay command involved. + Some(site) + if let Some(duplicate) = + crate::agents::pi::doctor::conflicting_extension_site(cwd, &site.path) => + { + Check { + name: NAME, + status: Status::Warn, + details: format!( + "two copies would load: {} and {}. pi de-duplicates by path, not by package, so both register hooks and every event is reported twice -- each turn is closed as superseded by its own duplicate, and the inline-shell gate decides one command twice. Keep one copy", + site.path.display(), + duplicate.display() + ), + } + } + // Installed, and switched off in pi's own settings. Same silent drop as the trust gate, + // from the other direction -- and `-e` ignores those filters, so the launcher still + // instruments a session the user's own `pi` runs are missing. + Some(site) if site.disabled_by_settings => Check { + name: NAME, + status: Status::Warn, + details: format!( + "{} is recorded in pi's settings with its extensions filtered off, so pi does not load it -- remove the `extensions` filter, or the `autoload: false`, on that entry. `nemo-relay run --agent pi` is unaffected: it passes `-e`, which applies no settings filter", + site.path.display() + ), + }, Some(site) => Check { name: NAME, status: Status::Pass, diff --git a/crates/cli/src/sessions/mod.rs b/crates/cli/src/sessions/mod.rs index 56a40195c..a45c4a24c 100644 --- a/crates/cli/src/sessions/mod.rs +++ b/crates/cli/src/sessions/mod.rs @@ -1622,6 +1622,11 @@ impl Session { // response drains it, so a rewrite recorded before a failing start would ride out on the // *next* response instead -- where the extension's `tool_call_id` echo reads it as // another call's rewrite and refuses that call. + // + // The failing branch has no test of its own, deliberately: `tool_call` is fallible only + // through process-global state, so inducing it would corrupt every other test in this + // binary. What is pinned instead is the invariant on the success side -- a rewrite reaches + // exactly one hook response, never the next one. self.tool_argument_transform = rewrite; Ok(()) } diff --git a/crates/cli/tests/coverage/agents/launcher_tests.rs b/crates/cli/tests/coverage/agents/launcher_tests.rs index 6590a19c1..b48886e37 100644 --- a/crates/cli/tests/coverage/agents/launcher_tests.rs +++ b/crates/cli/tests/coverage/agents/launcher_tests.rs @@ -1789,6 +1789,49 @@ fn pi_launch_finds_a_user_scope_install_without_an_environment_variable() { ); } +// `-e` adds to pi's set rather than replacing it, and pi de-duplicates by path, not by package, +// so a second distinct copy loads beside the launched one and every hook is posted twice -- each +// turn closed as superseded by its own duplicate. Nothing suppresses discovery from here without +// also dropping the user's own extensions, so the launch is refused and both copies are named. +#[test] +fn pi_launch_refuses_when_two_copies_would_load() { + let _guard = current_dir_lock().lock().unwrap(); + let temp = tempfile::tempdir().unwrap(); + let explicit = write_relay_pi_package(&temp.path().join("checkout")); + let agent_dir = temp.path().join("agent"); + let installed = agent_dir.join("extensions").join("nemo-relay"); + write_relay_pi_package(&installed); + let _env = EnvScope::set(&[ + ( + crate::agents::pi::launch::PI_EXTENSION_PATH_ENV, + Some(explicit.as_os_str()), + ), + ( + crate::agents::pi::doctor::PI_AGENT_DIR_ENV, + Some(agent_dir.as_os_str()), + ), + ]); + + let prepared = PreparedAgentLaunch::new( + CodingAgent::Pi, + vec!["pi".into()], + "http://127.0.0.1:4040", + &ResolvedConfig::default(), + false, + ); + + let Err(error) = prepared else { + panic!("a second distinct copy must refuse the launch"); + }; + let message = error.to_string(); + // Both paths, because the user has to pick one and cannot without knowing where they are. + assert!( + message.contains(&explicit.display().to_string()) + && message.contains(&installed.display().to_string()), + "the launch error should name both copies: {message}" + ); +} + // `-e` is never trust-gated. Promoting a project-scoped install to it would run // repository code pi itself declined to trust, which is the failure the preflight // exists to warn about rather than to work around. diff --git a/crates/cli/tests/coverage/agents/pi_doctor_tests.rs b/crates/cli/tests/coverage/agents/pi_doctor_tests.rs index 0c48247ed..b4c537da0 100644 --- a/crates/cli/tests/coverage/agents/pi_doctor_tests.rs +++ b/crates/cli/tests/coverage/agents/pi_doctor_tests.rs @@ -329,6 +329,171 @@ fn the_launch_path_skips_an_installed_source_that_is_not_a_path() { assert!(launchable_extension_path(temp.path()).is_none()); } +// pi's own configuration selector rewrites a string entry into the object form the +// moment a user toggles any resource of that package, so this shape is not a hand +// edit -- and while only strings were read, one keystroke in pi's UI made doctor +// report an installed extension as missing and made the launcher refuse to start. +#[test] +fn an_object_form_package_entry_is_found() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + let checkout = temp.path().join("checkout"); + write_relay_package(&checkout); + std::fs::write( + agent_dir.join("settings.json"), + r#"{"packages": [{"source": "../checkout"}]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + assert_eq!(sites.len(), 1, "{sites:?}"); + assert_eq!(sites[0].scope, ExtensionScope::User); + assert!(!sites[0].disabled_by_settings); + // Compared canonically: a recorded source is relative to the settings file, so the + // resolved path keeps the `..` pi itself would resolve away. + assert_eq!( + launchable_extension_path(temp.path()).map(|path| std::fs::canonicalize(path).unwrap()), + Some(std::fs::canonicalize(&checkout).unwrap()) + ); +} + +// A non-empty pattern list is matched against the package manifest, which this +// module does not read. Reporting it as loaded is the deliberate direction: a false +// negative here is the exact failure the whole module exists to prevent. +#[test] +fn an_object_form_entry_with_extension_patterns_is_still_found() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("checkout")); + std::fs::write( + agent_dir.join("settings.json"), + r#"{"packages": [{"source": "../checkout", "extensions": ["index.ts"], "skills": []}]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + assert_eq!(sites.len(), 1, "{sites:?}"); + assert!(!sites[0].disabled_by_settings); +} + +// Installed and switched off is not the same as absent, and must not be reported as +// either a plain Pass or a missing install. The launch path deliberately still uses +// it: `-e` applies no settings filter. +#[test] +fn an_object_form_entry_whose_extensions_are_disabled_is_reported_as_disabled() { + for body in [ + r#"{"packages": [{"source": "../checkout", "extensions": []}]}"#, + r#"{"packages": [{"source": "../checkout", "autoload": false}]}"#, + ] { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + let checkout = temp.path().join("checkout"); + write_relay_package(&checkout); + std::fs::write(agent_dir.join("settings.json"), body).unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + assert_eq!(sites.len(), 1, "{body}: {sites:?}"); + assert!(sites[0].disabled_by_settings, "{body}"); + // Still launchable, deliberately: `-e` applies no settings filter, so the launcher + // instruments a session the user's own `pi` runs are missing. + assert_eq!( + launchable_extension_path(temp.path()).map(|path| std::fs::canonicalize(path).unwrap()), + Some(std::fs::canonicalize(&checkout).unwrap()), + "{body}" + ); + } +} + +// Two ungated copies can be live at once -- a variable someone set at a checkout +// months ago, and the user-scope install the README recommends. Both are reported, +// because the order is the contract the launch path reads. +#[test] +fn a_distinct_explicit_copy_and_a_user_install_are_both_reported_explicit_first() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + let installed = agent_dir.join("extensions").join("nemo-relay"); + write_relay_package(&installed); + let explicit = temp.path().join("checkout"); + write_relay_package(&explicit); + + let _env = scoped(Some(explicit.as_os_str()), Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + assert_eq!(sites.len(), 2, "{sites:?}"); + assert_eq!(sites[0].scope, ExtensionScope::Explicit); + assert_eq!(sites[0].path, explicit); + assert_eq!(sites[1].scope, ExtensionScope::User); + + // And that is exactly the case pi cannot see: it de-duplicates by path, so both + // load, and every hook is posted twice. + assert_eq!( + conflicting_extension_site(temp.path(), &explicit), + Some(installed) + ); +} + +// One install is reachable both as its directory and as the entry file inside it, +// and pi resolves both to the same file through the `pi.extensions` manifest. That +// is one copy, and refusing to launch it would be a false alarm on the setup the +// launcher itself produces. +#[test] +fn the_same_install_reached_two_ways_is_not_a_second_copy() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + let installed = agent_dir.join("extensions").join("nemo-relay"); + write_relay_package(&installed); + let entry = installed.join("index.ts"); + + let _env = scoped(Some(entry.as_os_str()), Some(agent_dir.as_os_str())); + assert_eq!(conflicting_extension_site(temp.path(), &entry), None); +} + +// pi canonicalizes with `realpathSync`, so a symlinked copy is one copy to pi and +// must be one copy here. +#[cfg(unix)] +#[test] +fn a_symlinked_copy_is_not_a_second_copy() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + let installed = agent_dir.join("extensions").join("nemo-relay"); + write_relay_package(&installed); + let link = temp.path().join("link"); + std::os::unix::fs::symlink(&installed, &link).unwrap(); + + let _env = scoped(Some(link.as_os_str()), Some(agent_dir.as_os_str())); + assert_eq!(conflicting_extension_site(temp.path(), &link), None); +} + +// pi loads a project-scoped copy only for a trusted project, so it is not reliably +// a second load -- and refusing on it would block launches that are fine. The trust +// warning already names it. +#[test] +fn a_project_scoped_copy_is_not_a_second_copy() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + let installed = agent_dir.join("extensions").join("nemo-relay"); + write_relay_package(&installed); + write_relay_package( + &temp + .path() + .join(".pi") + .join("extensions") + .join("nemo-relay"), + ); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + assert_eq!(conflicting_extension_site(temp.path(), &installed), None); +} + // A user-scope install is what the README's install routes produce, and none of // them set an environment variable -- so the launcher has to find one without it. // An explicit path still wins, because someone who set it meant it. diff --git a/crates/cli/tests/coverage/shared/server_tests.rs b/crates/cli/tests/coverage/shared/server_tests.rs index 498bc2d74..4c6780e9e 100644 --- a/crates/cli/tests/coverage/shared/server_tests.rs +++ b/crates/cli/tests/coverage/shared/server_tests.rs @@ -4804,6 +4804,76 @@ async fn pi_tool_call_hook_evaluates_conditional_guardrails_once_when_an_interce ); } +// The rewrite is drained by the response that carries it and never rides out on the next one. That +// is the invariant behind publishing it only after the tool start can no longer fail: the +// extension's `tool_call_id` echo reads a stale rewrite as another call's and refuses that call, so +// a leak here blocks an unrelated tool rather than merely mis-recording one. +#[tokio::test] +async fn pi_tool_call_hook_hands_a_rewrite_to_one_response_only() { + let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; + let _ = deregister_tool_request_intercept("cli-pi-drain-once"); + register_tool_request_intercept( + "cli-pi-drain-once", + 1, + false, + Arc::new(|_name: String, args: Value| { + Box::pin(async move { + let mut args = args; + if let Some(object) = args.as_object_mut() + && object.get("path").and_then(Value::as_str) == Some("/work/first") + { + object.insert("path".into(), json!("/work/rewritten")); + } + Ok(args) + }) + }), + ) + .unwrap(); + let _cleanup = ToolInterceptCleanup("cli-pi-drain-once"); + + let app = router(test_config()); + let post = |call_id: &'static str, path: &'static str| { + let app = app.clone(); + async move { + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/hooks/pi") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "session_id": "pi-drain-session", + "hook_event_name": "tool_call", + "tool_call_id": call_id, + "tool_name": "read", + "input": { "path": path } + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + serde_json::from_slice::(&bytes).unwrap() + } + }; + + let rewritten = post("call-first", "/work/first").await; + assert_eq!(rewritten["tool_call"]["tool_call_id"], json!("call-first")); + assert_eq!( + rewritten["tool_call"]["input"], + json!({ "path": "/work/rewritten" }) + ); + + // A second, unrelated call the intercept does not touch. If the first rewrite were still on + // the session it would surface here under the wrong id, and the extension would refuse it. + let untouched = post("call-second", "/work/second").await; + assert_eq!(untouched, json!({}), "a drained rewrite must not reappear"); +} + // An unchanged call must keep returning the bare `{}` an allow has always been, or every existing // extension would start seeing a payload it has no contract for. #[tokio::test] diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index e0456b4e8..4045b9188 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -72,8 +72,10 @@ cp -r integrations/pi ~/.pi/agent/extensions/nemo-relay pi install /path/to/NeMo-Relay/integrations/pi ``` -`nemo-relay run --agent pi` needs neither — it passes the extension with `-e`, -which loads first and is never trust-gated. +`nemo-relay run --agent pi` resolves either route too — or +`NEMO_RELAY_PI_EXTENSION`, if you set it — and passes what it finds to `-e`, +which loads first and is never trust-gated. It needs one of them: it never +promotes a project-scoped install to `-e`. **Do not install from a git URL.** pi has no subdirectory syntax for a git @@ -116,7 +118,7 @@ configuration. The launcher resolves the extension from the same places `nemo-relay doctor pi` looks — `NEMO_RELAY_PI_EXTENSION` first, then a user-scope install — so either -install route below is enough and no variable is required. Unlike Claude Code and +install route above is enough and no variable is required. Unlike Claude Code and Codex, there is no Relay-managed install location to fall back on, because pi extensions live in the user's own configuration directories. diff --git a/integrations/pi/README.md b/integrations/pi/README.md index dcb00941a..df688f9db 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -105,7 +105,7 @@ Run it first whenever Relay does not seem to be doing anything. | Variable | Default | Meaning | |---|---|---| -| `NEMO_RELAY_PI_EXTENSION` | unset | Overrides where the launcher looks for this extension. Set by the launcher from what it resolved | +| `NEMO_RELAY_PI_EXTENSION` | unset | The first place the launcher looks for this extension. Ignored unless the path exists **and** its `package.json` names `nemo-relay-pi`, in which case resolution falls through to a user-scope install. Set by the launcher from what it resolved | | `NEMO_RELAY_PI_GATEWAY_URL` | `http://127.0.0.1:4040` | Gateway base URL | | `NEMO_RELAY_PI_TIMEOUT_MS` | `5000` | Per-request timeout | | `NEMO_RELAY_PI_FAIL` | `open` | `closed` blocks tool calls and inline shell commands when the gateway is unreachable | diff --git a/integrations/pi/index.ts b/integrations/pi/index.ts index cd2accb76..104380d1f 100644 --- a/integrations/pi/index.ts +++ b/integrations/pi/index.ts @@ -41,7 +41,7 @@ * All per-call state is keyed by `toolCallId`, which is the only correlator * pi provides. * - * Load it with `pi -e `, or let `nemo-relay launch pi` do it. + * Load it with `pi -e `, or let `nemo-relay run --agent pi` do it. * * **Model redirection.** pi resolves a base URL per model from a generated * catalog, so the extension points the active model's provider at the gateway From a16c9659f81978abbe3c148aba17580994dd3533 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 11:50:09 -0700 Subject: [PATCH 31/41] fix(pi): send the launcher's proxy credential on a redirected provider End-to-end testing found `nemo-relay run --agent pi` returning 401 on every model call the moment redirection fires -- the exact path the docs advertise with "Redirected; LLM spans appear under the turn". A gateway started by `run` authenticates its own client before a request intercept can rewrite the route, and rejects a provider call that does not present this invocation's credential. Claude Code's launcher injects it through `ANTHROPIC_CUSTOM_HEADERS`; Codex reads it through its `env_http_headers` provider configuration. `prepare_launch` already exports `NEMO_RELAY_PROXY_CREDENTIAL` for *every* agent, so the value was sitting in pi's environment the whole time -- the extension simply never read it, and `registerProvider` set only the session join key. It goes on the registration, beside the session id, for the same structural reason and a stronger one: the credential authenticates this invocation, so a provider the gateway does not front must never see it, and `registerProvider` runs only on a redirect. Absent -- a standalone `nemo-relay --bind` daemon requires no credential -- the key is omitted rather than sent empty. Not a `NEMO_RELAY_PI_*` name on purpose: the launcher exports one variable for all three agents, and a pi-specific alias would be a second name for one value. Verified: the new test fails against the previous source. The gateway probe is unaffected -- only provider passthrough is authenticated, not `/hooks/pi`. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- docs/nemo-relay-cli/pi.mdx | 7 ++++ integrations/pi/README.md | 1 + integrations/pi/index.ts | 21 ++++++++---- integrations/pi/src/provider-redirect.ts | 20 +++++++++++ integrations/pi/test/lifecycle.test.mjs | 42 ++++++++++++++++++++++++ 5 files changed, 85 insertions(+), 6 deletions(-) diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index 4045b9188..b8b0739e3 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -126,6 +126,13 @@ A **project-scoped** install is deliberately never used. `-e` is not trust-gated so passing one would load repository code pi itself declined to trust. Launch fails instead, and names the install routes that are not gated. +The launcher also passes this invocation's proxy credential through +`NEMO_RELAY_PROXY_CREDENTIAL`, and the extension sends it as +`x-nemo-relay-proxy-token` on a redirected provider. A gateway started by `run` +authenticates its own client before any intercept can rewrite the route, so +without it a redirect would succeed and every model call would come back `401`. A +standalone `nemo-relay --bind` daemon sets no credential and requires none. + Inspect what would be launched without starting pi: ```bash diff --git a/integrations/pi/README.md b/integrations/pi/README.md index df688f9db..06b70019e 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -112,6 +112,7 @@ Run it first whenever Relay does not seem to be doing anything. | `NEMO_RELAY_PI_REDIRECT` | `match` | `force` redirects without checking the upstream; `off` disables redirection | | `NEMO_RELAY_PI_OPENAI_UPSTREAM` | unset | What the gateway forwards OpenAI-compatible traffic to. Set by the launcher | | `NEMO_RELAY_PI_ANTHROPIC_UPSTREAM` | unset | What the gateway forwards Anthropic traffic to. Set by the launcher | +| `NEMO_RELAY_PROXY_CREDENTIAL` | unset | This invocation's proxy credential, set by `nemo-relay run` for every agent it starts. Sent as `x-nemo-relay-proxy-token` on a redirected provider only — a gateway the launcher started rejects a provider call without it. A standalone `nemo-relay --bind` daemon sets none and needs none | ## How Tool Gating Works diff --git a/integrations/pi/index.ts b/integrations/pi/index.ts index 104380d1f..75673d66e 100644 --- a/integrations/pi/index.ts +++ b/integrations/pi/index.ts @@ -266,20 +266,29 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { ); if (decision.kind === 'redirect') { // `baseUrl` rewrites the URL of every existing model for this provider and - // keeps their API and costs. `headers` is the session join key. + // keeps their API and costs. `headers` carries two things the gateway needs + // from a redirected call: the session join key, and -- when the launcher + // started this gateway -- its transparent-proxy credential. // - // It goes here rather than in a `before_provider_headers` handler because + // Both go here rather than in a `before_provider_headers` handler because // that hook is **global and carries no request identity**: its event has // only the headers, and its context is freshly built, so `ctx.model` is // whatever is selected *now*, not what this request is for. Scoping on it // gets both directions wrong -- omitting the key from a redirected call // whose model was captured before a switch, and leaking an internal session - // id to a third-party provider we deliberately did not redirect. Attaching - // it to the registration makes the scope structural: only providers we - // actually pointed at the gateway ever send it. + // id, or the credential, to a third-party provider we deliberately did not + // redirect. Attaching them to the registration makes the scope structural: + // only providers we actually pointed at the gateway ever send either. + // + // Without the credential the redirect succeeds and every model call then + // comes back 401 -- a gateway started by `nemo-relay run` authenticates its + // own client before any intercept can rewrite the route. pi.registerProvider(decision.provider, { baseUrl: redirect.gatewayUrl, - headers: { 'x-nemo-relay-session-id': sessionKey }, + headers: { + 'x-nemo-relay-session-id': sessionKey, + ...(redirect.proxyToken ? { 'x-nemo-relay-proxy-token': redirect.proxyToken } : {}), + }, }); redirectedProviders.add(decision.provider); } diff --git a/integrations/pi/src/provider-redirect.ts b/integrations/pi/src/provider-redirect.ts index 4d7ad342f..605bdfdb3 100644 --- a/integrations/pi/src/provider-redirect.ts +++ b/integrations/pi/src/provider-redirect.ts @@ -64,6 +64,20 @@ export type RedirectConfig = { * disables redirection entirely. */ mode: 'match' | 'force' | 'off'; + /** + * This invocation's transparent-proxy credential, when the launcher set one. + * + * A gateway started by `nemo-relay run` authenticates its own client before a + * request intercept can rewrite the route, and rejects a provider call that + * does not present it (`crates/cli/src/provider_auth.rs`). Hook posts are not + * covered by that check -- only provider passthrough is -- so without this the + * redirect succeeds and then every model call comes back 401, which is the one + * outcome the redirect exists to produce spans for. + * + * Absent for a standalone `nemo-relay --bind` daemon, which requires no + * credential, so the key is omitted rather than sent empty. + */ + proxyToken?: string; }; /** A model, narrowed to the fields redirection depends on. */ @@ -278,10 +292,16 @@ export function redirectConfigFromEnv(gatewayUrl: string): RedirectConfig { // the same as an absent key. const openaiUpstream = process.env.NEMO_RELAY_PI_OPENAI_UPSTREAM; const anthropicUpstream = process.env.NEMO_RELAY_PI_ANTHROPIC_UPSTREAM; + // Not a `NEMO_RELAY_PI_*` name, deliberately: the launcher exports this one for + // every agent it starts, and Codex reads the same variable through its + // `env_http_headers` provider configuration. A pi-specific alias would be a + // second name for one value, and the two could drift. + const proxyToken = process.env.NEMO_RELAY_PROXY_CREDENTIAL; return { gatewayUrl, mode, ...(openaiUpstream ? { openaiUpstream } : {}), ...(anthropicUpstream ? { anthropicUpstream } : {}), + ...(proxyToken ? { proxyToken } : {}), }; } diff --git a/integrations/pi/test/lifecycle.test.mjs b/integrations/pi/test/lifecycle.test.mjs index 4b7aa3564..d33e6b011 100644 --- a/integrations/pi/test/lifecycle.test.mjs +++ b/integrations/pi/test/lifecycle.test.mjs @@ -614,6 +614,7 @@ describe('the session join key on redirected providers', () => { beforeEach(() => { ctx.posts.length = 0; + delete process.env.NEMO_RELAY_PROXY_CREDENTIAL; }); const model = { @@ -679,6 +680,47 @@ describe('the session join key on redirected providers', () => { assert.equal(registrations[1].config.headers['x-nemo-relay-session-id'], 'sess-two'); }); + // A gateway started by `nemo-relay run` authenticates its own client before any + // intercept can rewrite the route, so a redirected call without this credential is + // rejected 401 -- the redirect succeeds and every model call then fails, which is + // the one outcome redirection exists to avoid. + it('carries the launcher proxy credential on a redirected provider', async () => { + process.env.NEMO_RELAY_PROXY_CREDENTIAL = 'nrp_testtoken'; + const { fire, registrations } = loadRecording('sess-one'); + await fire('session_start', { reason: 'startup' }); + + assert.equal(registrations.length, 1); + assert.equal(registrations[0].config.headers['x-nemo-relay-proxy-token'], 'nrp_testtoken'); + }); + + // A standalone `nemo-relay --bind` daemon requires no credential, so the launcher + // sets nothing. Sending the header empty would be a credential claim we cannot back. + it('omits the proxy credential header when the launcher set none', async () => { + const { fire, registrations } = loadRecording('sess-one'); + await fire('session_start', { reason: 'startup' }); + + assert.equal(registrations.length, 1); + assert.ok( + !('x-nemo-relay-proxy-token' in registrations[0].config.headers), + 'an absent credential must not become an empty header', + ); + }); + + // Same structural scope as the session id, and it matters more here: the credential + // authenticates *this* invocation, so a provider the gateway does not front must + // never see it. `registerProvider` runs only on a redirect, which is what enforces it. + it('never reaches a provider that was not redirected', async () => { + process.env.NEMO_RELAY_PROXY_CREDENTIAL = 'nrp_testtoken'; + process.env.NEMO_RELAY_PI_OPENAI_UPSTREAM = 'https://elsewhere.example/v1'; + try { + const { fire, registrations } = loadRecording('sess-one'); + await fire('session_start', { reason: 'startup' }); + assert.equal(registrations.length, 0, 'a mismatched upstream must not register'); + } finally { + process.env.NEMO_RELAY_PI_OPENAI_UPSTREAM = 'https://api.openai.com/v1'; + } + }); + it('does not re-register when the session id has not moved', async () => { const { fire, registrations } = loadRecording('sess-one'); await fire('session_start', { reason: 'startup' }); From e7a6943ddd4263bef8975ca4c65154885883394f Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 12:02:32 -0700 Subject: [PATCH 32/41] fix(pi): fold preflight findings into the status, configured or not `nemo-relay agents --json` reported `status: "pass"` for a pi whose only install is project-scoped -- the exact silent skip the preflight exists to catch -- while its own nested check said `warn`. The fold was gated on `configured || target_requested`, and `configured` means only that `[agents.pi] command` is set in Relay config, which almost nobody sets. That gate is right for readiness ("the hook config is missing" should not make a bare `doctor` complain about an agent you do not run) and wrong for a preflight finding, which is *evidence*: every warning branch already requires the extension to be installed on this machine, and a machine without one reports `Info`, which folds either way. `doctor pi --json` was already correct; now `agents --json` agrees with it. Also documented, from the same end-to-end round: - **A slow gateway multiplies, it does not just delay.** Posts are serialized by design, so a gating hook waits out everything queued ahead of it: against a gateway that holds requests, the first gate of a session pays `NEMO_RELAY_PI_TIMEOUT_MS` once per queued post, not once. The queue stays -- it is what keeps session and turn boundaries derivable from arrival order -- but the cost now appears where the value is chosen. - **The shipped `rust-native-plugin` example blocks every pi tool call.** Its intercept tags arguments with two added keys, and an added key is exactly what the shape invariant refuses. Correct on both sides, and previously written down on neither, so it is noted in the pi transform section and in the example's own README. The example is deliberately not changed: adding keys is what it exists to demonstrate. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- crates/cli/src/diagnostics/mod.rs | 23 +++++++++++++-- .../cli/tests/coverage/shared/doctor_tests.rs | 29 +++++++++++++++++++ docs/nemo-relay-cli/pi.mdx | 11 ++++++- examples/rust-native-plugin/README.md | 5 ++++ integrations/pi/README.md | 2 +- 5 files changed, 65 insertions(+), 5 deletions(-) diff --git a/crates/cli/src/diagnostics/mod.rs b/crates/cli/src/diagnostics/mod.rs index de1116308..33f124083 100644 --- a/crates/cli/src/diagnostics/mod.rs +++ b/crates/cli/src/diagnostics/mod.rs @@ -466,9 +466,7 @@ async fn collect_agent( ); let checks = agent_preflight_checks(agent, probe_mode, configured || target_requested, resolved).await; - for check in &checks { - status = combine_status(status, check.status, configured || target_requested); - } + status = fold_preflight_checks(status, &checks); AgentInfo { name: agent.as_arg(), status, @@ -783,6 +781,25 @@ fn agent_command_status(path: Option<&Path>, configured: bool, target_requested: } } +/// Fold preflight findings into the agent status, configured or not. +/// +/// The readiness gate `combine_status` applies elsewhere asks whether the user has +/// told Relay to use this agent, which is the right question for "the hook config +/// is missing" -- nobody wants a bare `doctor` complaining about an agent they do +/// not run. It is the wrong question here, because a preflight finding is +/// *evidence* rather than readiness: every warning branch already requires the +/// extension to be installed on this machine, and a machine without one reports +/// `Info`, which never folds either way. +/// +/// Gating on `configured` meant `agents --json` reported `status: "pass"` for a pi +/// whose only install is project-scoped -- the exact silent skip the check exists +/// to catch -- while its own nested check said `warn`. +fn fold_preflight_checks(status: Status, checks: &[Check]) -> Status { + checks.iter().fold(status, |status, check| { + combine_status(status, check.status, true) + }) +} + fn combine_status(base: Status, hook: Status, readiness_required: bool) -> Status { if matches!(base, Status::Fail) || matches!(hook, Status::Fail) { return Status::Fail; diff --git a/crates/cli/tests/coverage/shared/doctor_tests.rs b/crates/cli/tests/coverage/shared/doctor_tests.rs index 65bce3289..4cbf3dd4f 100644 --- a/crates/cli/tests/coverage/shared/doctor_tests.rs +++ b/crates/cli/tests/coverage/shared/doctor_tests.rs @@ -2475,6 +2475,35 @@ fn format_agents_json_matches_doctor_agents_shape() { ); } +// A preflight finding is evidence, not readiness. Gating the fold on `configured` made +// `agents --json` report `pass` for a pi whose only install is project-scoped -- the exact silent +// skip the check exists to catch -- while the nested check said `warn`. The `Info` half is what +// keeps a bare `doctor` quiet on a machine with no pi extension at all. +#[test] +fn preflight_findings_fold_into_the_agent_status_whether_or_not_it_is_configured() { + let warn = [Check { + name: "pi extension load path", + status: Status::Warn, + details: "project-scoped".into(), + }]; + assert_eq!(fold_preflight_checks(Status::Pass, &warn), Status::Warn); + + let info = [Check { + name: "pi extension load path", + status: Status::Info, + details: "not located".into(), + }]; + assert_eq!(fold_preflight_checks(Status::Pass, &info), Status::Pass); + + let fail = [Check { + name: "pi extension load path", + status: Status::Fail, + details: "unreadable".into(), + }]; + assert_eq!(fold_preflight_checks(Status::Pass, &fail), Status::Fail); + assert_eq!(fold_preflight_checks(Status::Pass, &[]), Status::Pass); +} + #[test] fn doctor_agent_status_helpers_cover_readiness_and_version_outcomes() { assert_eq!( diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index b8b0739e3..50715b370 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -40,7 +40,11 @@ Three consequences follow from that shape: - The extension pays a round trip per gated tool call. pi awaits extension handlers on its critical path, so a blocking hook is synchronous by construction. Observability-only hooks are queued rather than awaited and - drained at shutdown, so they do not charge that path. + drained at shutdown, so they do not charge that path — while the gateway + answers. Posts go out one at a time in hook order, so a gating hook also waits + for whatever is queued ahead of it: against a gateway that holds requests, the + first gate of a session pays `NEMO_RELAY_PI_TIMEOUT_MS` once per queued post, + not once. - Model traffic is redirected by the extension, not by configuration. pi resolves `baseUrl` per model from a generated catalog and has no base-URL flag or generic environment override, so the extension calls @@ -233,6 +237,11 @@ A transform that violates the invariant **blocks the call**, with a reason stating that a policy could not be applied rather than that the request was refused. Running the original arguments would silently discard the policy. +The shipped `examples/rust-native-plugin` demonstrates the mechanism rather than +a policy: its tool request intercept tags arguments with `native_plugin_tag` and +`native_tool_request_intercept`, which adds keys and so blocks every pi tool +call. Enable it to see the refusal, not as a template to copy. + ## Inline Shell Gating pi's bang prefix runs a command outside the tool registry: `!git status` runs it diff --git a/examples/rust-native-plugin/README.md b/examples/rust-native-plugin/README.md index 790605ffa..acb0c5b0a 100644 --- a/examples/rust-native-plugin/README.md +++ b/examples/rust-native-plugin/README.md @@ -115,6 +115,11 @@ The example registers the following runtime behavior: Native plugins are not sandboxed. They run in the Relay process and must not unwind across ABI callbacks. +The tool request intercept **adds** keys to a tool's arguments. That is fine in +process, but the pi extension accepts only a rewrite that preserves the argument +shape, so running pi against a gateway with this example enabled blocks every +tool call. See the argument-transform notes in `docs/nemo-relay-cli/pi.mdx`. + Request intercepts do not own an LLM lifecycle because they run before Relay creates the LLM scope. `register_llm_request_intercept` returns one `LlmRequestInterceptOutcome`, whose `pending_marks` Relay emits in interceptor diff --git a/integrations/pi/README.md b/integrations/pi/README.md index 06b70019e..98c5e2777 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -107,7 +107,7 @@ Run it first whenever Relay does not seem to be doing anything. |---|---|---| | `NEMO_RELAY_PI_EXTENSION` | unset | The first place the launcher looks for this extension. Ignored unless the path exists **and** its `package.json` names `nemo-relay-pi`, in which case resolution falls through to a user-scope install. Set by the launcher from what it resolved | | `NEMO_RELAY_PI_GATEWAY_URL` | `http://127.0.0.1:4040` | Gateway base URL | -| `NEMO_RELAY_PI_TIMEOUT_MS` | `5000` | Per-request timeout | +| `NEMO_RELAY_PI_TIMEOUT_MS` | `5000` | Per-request timeout. Posts are serialized, so a gating hook also waits out everything queued ahead of it — an unresponsive gateway costs this once per queued post, not once | | `NEMO_RELAY_PI_FAIL` | `open` | `closed` blocks tool calls and inline shell commands when the gateway is unreachable | | `NEMO_RELAY_PI_REDIRECT` | `match` | `force` redirects without checking the upstream; `off` disables redirection | | `NEMO_RELAY_PI_OPENAI_UPSTREAM` | unset | What the gateway forwards OpenAI-compatible traffic to. Set by the launcher | From 01411c19175ddd7b7701b2532799a92e62965a42 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 14:06:51 -0700 Subject: [PATCH 33/41] fix(pi): a gateway that answered was reached; say which fault happened `resolveFault` opened every fail-closed block with "could not be reached", including the four cases where the gateway did answer -- HTTP 413 or 500, a 403 without the guardrail marker, an unparseable 2xx body -- and the case where the gateway was never consulted because the inline-shell handler itself threw. The string reaches the model, and the user, verbatim. Live against a gateway returning 413 it read "could not be reached ... Details: gateway returned HTTP 413": the `Details:` line was right and the sentence sent the reader to debug a socket that was working. A fault now carries `reached`, and the opening picks from it. The tail is unchanged, because it is the part a model has to act on and it is the same either way: nothing judged the request, so the request is not what to change. The same round asked for tool arguments to be bounded the way results are. **Not done, because it would be unsafe.** The `tool_call` post is the gated one: a guardrail decides on exactly those arguments, and a request intercept sends a rewritten copy back for pi to execute. The shape invariant checks JSON types and key sets, not content -- so a truncated `content` passes it and a `write` lands on disk cut short. A result has no path back into execution, which is why only results are bounded. That asymmetry, and the 20 MiB gateway ceiling that is the real bound on arguments, are now written down, and a test pins that the invariant cannot tell a shortened string from the original. Also corrects a claim in the transform test header that this PR had already retracted eleven commits earlier: the tool schema is reachable, and not using it is a choice about staleness rather than a limitation. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- "0\"" | 1 + "102\"" | 1 + "12\"" | 1 + "1206\"" | 1 + docs/nemo-relay-cli/pi.mdx | 4 +- integrations/pi/README.md | 25 ++++++++-- integrations/pi/index.ts | 8 ++-- integrations/pi/src/gateway-client.ts | 47 +++++++++++++++---- .../pi/test/argument-transform.test.mjs | 28 ++++++++--- integrations/pi/test/gateway-client.test.mjs | 27 ++++++++++- 10 files changed, 119 insertions(+), 24 deletions(-) create mode 100644 "0\"" create mode 100644 "102\"" create mode 100644 "12\"" create mode 100644 "1206\"" diff --git "a/0\"" "b/0\"" new file mode 100644 index 000000000..edea114c6 --- /dev/null +++ "b/0\"" @@ -0,0 +1 @@ +"/Users/yuchenz/Desktop/Work/Project/NeMo-Relay/target/debug/deps/nemo_relay-05557a0b1c85a83a - diff --git "a/102\"" "b/102\"" new file mode 100644 index 000000000..c490ea491 --- /dev/null +++ "b/102\"" @@ -0,0 +1 @@ +"/Users/yuchenz/Desktop/Work/Project/NeMo-Relay/target/debug/deps/cli_tests-2f35b16f0798cdf9 - diff --git "a/12\"" "b/12\"" new file mode 100644 index 000000000..ea9391eea --- /dev/null +++ "b/12\"" @@ -0,0 +1 @@ +"/Users/yuchenz/Desktop/Work/Project/NeMo-Relay/target/debug/deps/architecture_tests-2096cbd4d612e5af - diff --git "a/1206\"" "b/1206\"" new file mode 100644 index 000000000..771012a08 --- /dev/null +++ "b/1206\"" @@ -0,0 +1 @@ +"/Users/yuchenz/Desktop/Work/Project/NeMo-Relay/target/debug/deps/nemo_relay_cli-40a232977d19ebc8 - diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index 50715b370..92dbfed1c 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -195,7 +195,9 @@ that gives up. The default is fail-open, so an unreachable gateway does not brick the agent. `NEMO_RELAY_PI_FAIL=closed` opts in to blocking, and the block reason then says explicitly that it is an infrastructure fault rather than a judgment about the -request. +request — and which fault it was, because a gateway that never answered and one +that answered without a decision (a rejected payload, an unreadable body, a 403 +with no guardrail marker) are debugged in different places. ## Argument Transforms diff --git a/integrations/pi/README.md b/integrations/pi/README.md index 98c5e2777..dd5446d56 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -227,7 +227,9 @@ which pi records exactly as if the command had run: `NEMO_RELAY_PI_FAIL` governs this path too: a gateway that cannot be reached allows the command by default, and refuses it under `closed` with a reason that -says explicitly that it is an infrastructure fault rather than a judgment. +says explicitly that it is an infrastructure fault rather than a judgment — and +whether the gateway never answered or answered without a decision, so the reader +is not sent to debug a socket that is working. **A rewritten command is refused, not run.** pi's `user_bash` result can replace the *result* or the execution backend, but never the command — both call sites @@ -373,8 +375,25 @@ the first event of the turn. **Tool results are truncated at 2000 characters** before they are forwarded, with the overflow replaced by a `... [truncated N chars]` suffix. The gateway therefore records what a tool returned, not necessarily all of it — a large file -read or a long command output is cut. This keeps hook payloads bounded; raise -`MAX_CONTENT_CHARS` in `index.ts` if a policy needs to see more. +read or a long command output is cut. Raise `MAX_CONTENT_CHARS` in `index.ts` if +a policy needs to see more. + +**Tool arguments are not truncated, and must not be.** The `tool_call` post is +the gated one: a guardrail decides on exactly those arguments, and a request +intercept can send a rewritten copy back for pi to execute. Truncating them would +mean deciding on text the tool will not run — and worse, the rewrite that comes +back would carry the truncation, because the shape invariant checks JSON types +and key sets rather than content, so a shortened `content` is applied verbatim +and a `write` lands on disk cut short. A result has no path back into execution, +which is why only results are bounded. + +The ceiling is therefore the gateway's, not the extension's: +`gateway.max_hook_payload_bytes`, 20 MiB by default. A post above it is rejected +with HTTP 413 before any event exists, so the call is decided by +`NEMO_RELAY_PI_FAIL` — under `open` it runs ungated, leaving a span synthesized +from `tool_execution_end` with no arguments. Model-authored arguments do not +approach that ceiling; if some tool ever does, raise the gateway limit rather +than cutting the arguments. **Subagents.** pi ships no nested-agent hook of its own — the extension has nothing to derive a subagent id from — so `subagent_start` / `subagent_end` are diff --git a/integrations/pi/index.ts b/integrations/pi/index.ts index 75673d66e..837043a70 100644 --- a/integrations/pi/index.ts +++ b/integrations/pi/index.ts @@ -505,7 +505,7 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { ); const decision = - outcome.kind === 'fault' ? resolveFault(active, outcome.detail, event.toolName) : outcome; + outcome.kind === 'fault' ? resolveFault(active, outcome, event.toolName) : outcome; if (decision.kind === 'block') return { block: true, reason: decision.reason }; @@ -639,7 +639,7 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { const decision = outcome.kind === 'fault' - ? resolveFault(active, outcome.detail, USER_BASH_TOOL_NAME) + ? resolveFault(active, outcome, USER_BASH_TOOL_NAME) : outcome; if (decision.kind === 'block') { @@ -678,7 +678,9 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { const detail = error instanceof Error ? error.message : String(error); const fault = resolveFault( config ?? configFromEnv(safeSessionId(ctx)), - `the inline-shell gate failed: ${detail}`, + // Not `reached`: this is the handler failing, not the gateway. Whatever the + // gateway said, no decision came out of it here. + { kind: 'fault', reached: false, detail: `the inline-shell gate failed: ${detail}` }, USER_BASH_TOOL_NAME, ); if (fault.kind === 'block') return { result: refusalResult(fault.reason) }; diff --git a/integrations/pi/src/gateway-client.ts b/integrations/pi/src/gateway-client.ts index 84c6bed3d..f85bc8fc8 100644 --- a/integrations/pi/src/gateway-client.ts +++ b/integrations/pi/src/gateway-client.ts @@ -29,7 +29,19 @@ export type HookOutcome = /** Allowed. `body` carries a rewritten payload when a request intercept produced one. */ | { kind: 'allow'; body?: { tool_call?: { tool_call_id?: unknown; input?: unknown } } } | { kind: 'block'; reason: string } - | { kind: 'fault'; detail: string }; + /** + * Neither a verdict nor a usable success. + * + * `reached` is false when nothing came back -- the connection failed, or nothing + * answered in time -- and true when the gateway did answer and the answer was not + * a decision, such as a rejected payload or a body that will not parse. Both block + * under `NEMO_RELAY_PI_FAIL=closed`, but they send whoever reads the block to two + * different places, so the reason has to say which one happened. + */ + | { kind: 'fault'; detail: string; reached: boolean }; + +/** The fault arm of {@link HookOutcome}, named so a caller can build one. */ +export type HookFault = Extract; export type GatewayConfig = { /** Base URL of the gateway, e.g. `http://127.0.0.1:4040`. */ @@ -91,7 +103,11 @@ export async function postHook( // is an infrastructure fault, resolved by `NEMO_RELAY_PI_FAIL` like any other. const body = await safeJson(response); if (body === null || typeof body !== 'object' || Array.isArray(body)) { - return { kind: 'fault', detail: 'gateway returned a success body that is not a JSON object' }; + return { + kind: 'fault', + reached: true, + detail: 'gateway returned a success body that is not a JSON object', + }; } return { kind: 'allow', body }; } @@ -104,16 +120,18 @@ export async function postHook( } // A 403 without the guardrail marker is an authorization fault, not a // policy decision; do not present it to the model as one. - return { kind: 'fault', detail: `gateway returned 403 without a guardrail reason` }; + return { kind: 'fault', reached: true, detail: `gateway returned 403 without a guardrail reason` }; } - return { kind: 'fault', detail: `gateway returned HTTP ${response.status}` }; + return { kind: 'fault', reached: true, detail: `gateway returned HTTP ${response.status}` }; } catch (error) { const detail = error instanceof Error && error.name === 'AbortError' ? `gateway did not respond within ${config.timeoutMs}ms` : `gateway request failed: ${error instanceof Error ? error.message : String(error)}`; - return { kind: 'fault', detail }; + // Nothing usable came back: a transport failure, or a timeout that may have arrived + // and never answered. Either way there is no response to have misread. + return { kind: 'fault', reached: false, detail }; } finally { clearTimeout(timer); } @@ -137,14 +155,25 @@ export function postAndForget( } /** Resolve a fault into an allow/block decision using the configured policy. */ -export function resolveFault(config: GatewayConfig, detail: string, toolName: string): HookOutcome { +export function resolveFault( + config: GatewayConfig, + fault: HookFault, + toolName: string, +): HookOutcome { if (config.onFault === 'open') return { kind: 'allow' }; + // Two openings, one tail. The tail is the part a model has to act on and it is the + // same either way: nothing judged the request, so the request is not what to change. + // The opening differs because "could not be reached", said of a gateway that replied + // 413, sends the reader to debug connectivity -- the one thing that is working. This + // string reaches the model verbatim, so it is also what the user reads. + const opening = fault.reached + ? `The NeMo Relay policy gateway answered this ${toolName} call without a usable decision` + : `The NeMo Relay policy gateway could not be reached to authorize this ${toolName} call`; return { kind: 'block', reason: - `The NeMo Relay policy gateway could not be reached to authorize this ${toolName} call, ` + - `so it was blocked rather than allowed through unchecked. This is an infrastructure fault, ` + - `not a judgment about the request. Details: ${detail}`, + `${opening}, so it was blocked rather than allowed through unchecked. This is an ` + + `infrastructure fault, not a judgment about the request. Details: ${fault.detail}`, }; } diff --git a/integrations/pi/test/argument-transform.test.mjs b/integrations/pi/test/argument-transform.test.mjs index 8cb97ce1f..cc2a5c88a 100644 --- a/integrations/pi/test/argument-transform.test.mjs +++ b/integrations/pi/test/argument-transform.test.mjs @@ -5,10 +5,12 @@ * The argument-transform decision matrix. * * pi validates tool arguments *before* the `tool_call` hook and never - * re-validates, and the extension cannot read the tool's schema, so a rewrite - * that violates the schema would execute. The shape check is what stands in for - * validation: same keys, same JSON types, recursively. These tests pin both the - * cases it must allow and the ones it must refuse. + * re-validates, so a rewrite that violates the schema would execute. The schema + * is reachable -- `pi.getAllTools()` exposes it -- and deliberately not used, + * because pi's tool set is per-session mutable and one read goes stale. The + * shape check is what stands in for validation instead: same keys, same JSON + * types, recursively. These tests pin both the cases it must allow and the ones + * it must refuse. * * Run: node --test integrations/pi/test/*.test.mjs */ @@ -112,12 +114,26 @@ describe('transform decision', () => { }); describe('what the shape check does not promise', () => { - // Documented limitation, asserted so it is not mistaken for validation: the extension cannot see - // the tool's schema, so pattern/enum/range violations pass the check and will execute. + // Documented limitation, asserted so it is not mistaken for validation: the check does not + // consult the tool's schema, so pattern/enum/range violations pass and will execute. it('allows a value the schema might still reject', () => { assert.equal(shapeViolation({ path: 'a.txt' }, { path: '../../etc/shadow' }), null); assert.equal(shapeViolation({ mode: 'read' }, { mode: 'not-an-enum-member' }), null); }); + + // The check is structural, not content-aware: a shortened string is still a string, so a + // truncated argument would come back rewritten and be applied verbatim -- a `write` would land + // on disk cut short. This is why the gated post forwards arguments whole while results are + // bounded at 2000 characters. See "What Is Not Represented" in the README. + it('cannot tell a shortened string from the original, which is why arguments are never truncated', () => { + assert.equal( + shapeViolation( + { path: '/work/a.txt', content: 'a much longer original body' }, + { path: '/work/a.txt', content: 'short' }, + ), + null, + ); + }); }); describe('applying the transform', () => { diff --git a/integrations/pi/test/gateway-client.test.mjs b/integrations/pi/test/gateway-client.test.mjs index fd72e725d..5877ef672 100644 --- a/integrations/pi/test/gateway-client.test.mjs +++ b/integrations/pi/test/gateway-client.test.mjs @@ -96,6 +96,7 @@ describe('gateway client wire contract', () => { const outcome = await postHook(baseConfig(url), { hook_event_name: name }); assert.equal(outcome.kind, 'fault', `${name} must not be a plain allow`); assert.match(outcome.detail, /not a JSON object/); + assert.equal(outcome.reached, true, `${name} answered; it was not unreachable`); } }); @@ -129,12 +130,14 @@ describe('gateway client wire contract', () => { // it as one would tell the model a policy considered and refused its call. const outcome = await postHook(baseConfig(url), { hook_event_name: 'naked-403' }); assert.equal(outcome.kind, 'fault'); + assert.equal(outcome.reached, true, 'a refusal is an answer'); }); it('reports a non-403 error status as a fault, not a block', async () => { const outcome = await postHook(baseConfig(url), { hook_event_name: 'boom' }); assert.equal(outcome.kind, 'fault'); assert.match(outcome.detail, /HTTP 500/); + assert.equal(outcome.reached, true); }); it('times out rather than hanging pi\'s critical path', async () => { @@ -143,6 +146,7 @@ describe('gateway client wire contract', () => { }); assert.equal(outcome.kind, 'fault'); assert.match(outcome.detail, /did not respond within 50ms/); + assert.equal(outcome.reached, false, 'nothing came back to have misread'); }); it('reports an unreachable gateway as a fault', async () => { @@ -151,6 +155,7 @@ describe('gateway client wire contract', () => { hook_event_name: 'tool_call', }); assert.equal(outcome.kind, 'fault'); + assert.equal(outcome.reached, false); }); it('sends the session id in both the header and the payload', async () => { @@ -168,7 +173,7 @@ describe('failure policy', () => { it('fails open by default so a dead sidecar does not brick the agent', () => { const outcome = resolveFault( { url: '', timeoutMs: 1, onFault: 'open', sessionId: 's' }, - 'connection refused', + { kind: 'fault', reached: false, detail: 'connection refused' }, 'read', ); assert.deepEqual(outcome, { kind: 'allow' }); @@ -177,13 +182,31 @@ describe('failure policy', () => { it('fails closed on request, and says the block is infrastructure not policy', () => { const outcome = resolveFault( { url: '', timeoutMs: 1, onFault: 'closed', sessionId: 's' }, - 'connection refused', + { kind: 'fault', reached: false, detail: 'connection refused' }, 'read', ); assert.equal(outcome.kind, 'block'); assert.match(outcome.reason, /infrastructure fault, not a judgment/); + assert.match(outcome.reason, /could not be reached/); assert.match(outcome.reason, /connection refused/); }); + + // A gateway that replied 413 was reached. Sending the reader to debug connectivity is + // the wrong search, and this sentence is what the model reads too, so it has to name + // the failure that actually happened. + it('says the gateway answered when it answered, rather than that it was unreachable', () => { + const outcome = resolveFault( + { url: '', timeoutMs: 1, onFault: 'closed', sessionId: 's' }, + { kind: 'fault', reached: true, detail: 'gateway returned HTTP 413' }, + 'write', + ); + assert.equal(outcome.kind, 'block'); + assert.match(outcome.reason, /answered this write call without a usable decision/); + assert.doesNotMatch(outcome.reason, /could not be reached/); + // The tail is unchanged: nothing judged the request either way. + assert.match(outcome.reason, /infrastructure fault, not a judgment/); + assert.match(outcome.reason, /HTTP 413/); + }); }); describe('configFromEnv', () => { From 9d6adb2bfa09a7859ab068308299d96d822ee7fe Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 14:16:40 -0700 Subject: [PATCH 34/41] fix(pi): model pi's real extension set, and stop naming the wrong closer Five findings, four of them on code the last two rounds added. **Both source readers stopped at the first match.** pi resolves every distinct package source -- two `packages` entries are two identities to `getPackageIdentity`, and `collectAutoExtensionEntries` walks a whole directory -- so two copies inside *one* source both load and post every hook twice, while the duplicate check saw one. Both readers now return every copy. The directory scan is restricted to the shapes pi accepts as an entry and sorted, because `read_dir` order is undefined and the launcher's choice must not vary run to run. **The conflict predicate modelled neither direction of pi's active set.** It counted copies pi's own settings switch off -- a hard launch refusal over a copy that was never going to register a hook -- and ignored project-scoped copies, which a *trusted* project does load beside `-e`. It now refuses only on copies pi is certain to load. The project case cannot be decided from here at all (`-a` overrides trust, `defaultProjectTrust` pre-answers it, session-only trust persists nothing), so it becomes a launch note rather than a refusal: refusing would block every launch in an untrusted project, and `-p`, `--mode json` and `--mode rpc` never prompt, so untrusted is the common state. **A non-empty `extensions` filter was read as "enabled".** pi's own configuration selector disables a resource by writing `-`, a force-exclude pi applies last and unconditionally -- so one keystroke in pi's UI left doctor reporting Pass for a package pi loads nothing from. Exact `+`/`-` patterns are decided now, against the entry points the manifest declares. Globs are not: pi expands those with `minimatch`, and a false warning costs more than a missing one, so anything undecidable still reads as enabled. **Every scope end now names its own closer.** `close_agent_scope` had no metadata channel while `close_turn_scope` did, so pi's session end repeated `session_start` and bucketing ATOF by `hook_event_name` never yielded a session end. Shutdown and sweep closes still pass `None`: no hook stands behind them. **A pi session that only ever held a mark is now swept.** `is_idle_for` required an open turn, because the sweeper's job is to close an idle *turn* -- but pi's marks open the session scope instead, which is exactly what `has_explicit_turn_start` is for, so those sessions were resident until process shutdown while Codex's and Claude Code's were swept. Narrowed by two guards: a session pi announced is a user idling between turns and is left alone, and `turn_index == 0` protects one whose `session_start` was lost but which did work. The shared non-object-payload question stays deferred -- it changes all three hook routes, and a plain `{}` defeats the obvious fix. Also repairs two doctor strings whose line continuations were lost when they were written, rendering with runs of stray spaces mid-sentence. Every new test was run against the previous source first; six of them fail there. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- crates/cli/src/agents/pi/doctor.rs | 289 +++++++++++++----- crates/cli/src/agents/pi/launch.rs | 18 ++ crates/cli/src/diagnostics/mod.rs | 42 +-- crates/cli/src/sessions/idle.rs | 4 +- crates/cli/src/sessions/mod.rs | 50 ++- .../tests/coverage/agents/pi_doctor_tests.rs | 104 ++++++- .../tests/coverage/shared/session_tests.rs | 87 ++++++ 7 files changed, 499 insertions(+), 95 deletions(-) diff --git a/crates/cli/src/agents/pi/doctor.rs b/crates/cli/src/agents/pi/doctor.rs index 1a4ab809e..cce061ac4 100644 --- a/crates/cli/src/agents/pi/doctor.rs +++ b/crates/cli/src/agents/pi/doctor.rs @@ -179,40 +179,44 @@ pub(crate) fn relay_extension_sites(cwd: &Path) -> Vec { disabled_by_settings: false, }); } - if let Some(dir) = user_extensions_dir() - && let Some(path) = relay_entry_in_directory(&dir) - { - sites.push(ExtensionSite { - path, - scope: ExtensionScope::User, - disabled_by_settings: false, - }); - } - if let Some(settings) = user_settings_path() - && let Some(install) = relay_package_in_settings(&settings) - { - sites.push(ExtensionSite { - path: install.path, - scope: ExtensionScope::User, - disabled_by_settings: install.disabled, - }); - } - if let Some(path) = relay_entry_in_directory(&cwd.join(PI_CONFIG_DIR).join("extensions")) { - sites.push(ExtensionSite { - path, - scope: ExtensionScope::Project, - disabled_by_settings: false, - }); + let discovered = |sites: &mut Vec, dir: &Path, scope| { + sites.extend( + relay_entries_in_directory(dir) + .into_iter() + .map(|path| ExtensionSite { + path, + scope, + disabled_by_settings: false, + }), + ); + }; + let recorded = |sites: &mut Vec, settings: &Path, scope| { + sites.extend( + relay_packages_in_settings(settings) + .into_iter() + .map(|install| ExtensionSite { + path: install.path, + scope, + disabled_by_settings: install.disabled, + }), + ); + }; + if let Some(dir) = user_extensions_dir() { + discovered(&mut sites, &dir, ExtensionScope::User); } - if let Some(install) = - relay_package_in_settings(&cwd.join(PI_CONFIG_DIR).join(PI_SETTINGS_FILE)) - { - sites.push(ExtensionSite { - path: install.path, - scope: ExtensionScope::Project, - disabled_by_settings: install.disabled, - }); + if let Some(settings) = user_settings_path() { + recorded(&mut sites, &settings, ExtensionScope::User); } + discovered( + &mut sites, + &cwd.join(PI_CONFIG_DIR).join("extensions"), + ExtensionScope::Project, + ); + recorded( + &mut sites, + &cwd.join(PI_CONFIG_DIR).join(PI_SETTINGS_FILE), + ExtensionScope::Project, + ); sites } @@ -266,12 +270,25 @@ pub(crate) fn launchable_extension_path(cwd: &Path) -> Option { /// /// Project scope is excluded: pi loads a project-scoped extension only for a /// trusted project, so it is not reliably a second load, and refusing on it would -/// block launches that are fine. The existing trust warning already names it. +/// block every launch in an untrusted project over a copy that will not run. The +/// launch note and the trust warning name it instead. +/// +/// A copy pi's own settings switch off is excluded for the opposite reason -- it is +/// reliably *not* a load. A filtered-off package's files are added with +/// `enabled: false` and dropped before the merge (`applyPackageFilter`, pi +/// `v0.84.0`, `core/package-manager.ts:2208`), so counting it here refused a launch +/// over a copy that was never going to register a hook. +/// +/// So this refuses only on copies pi is certain to load, and never on one it merely +/// might. The remaining false negative -- a trusted project holding a second copy -- +/// is a doubled trace with a note rather than a blocked launch, which is the right +/// side to be wrong on: `-p`, `--mode json` and `--mode rpc` never prompt for trust, +/// so untrusted is the common state. pub(crate) fn conflicting_extension_site(cwd: &Path, launched: &Path) -> Option { let launched_root = package_root(launched); relay_extension_sites(cwd) .into_iter() - .filter(|site| site.scope != ExtensionScope::Project) + .filter(|site| site.scope != ExtensionScope::Project && !site.disabled_by_settings) .find(|site| package_root(&site.path) != launched_root) .map(|site| site.path) } @@ -289,19 +306,43 @@ fn package_root(path: &Path) -> Option { canonical.parent().map(Path::to_path_buf) } -/// The NeMo Relay extension inside a pi auto-discovery directory, if it is there. +/// Every copy of the NeMo Relay extension inside a pi auto-discovery directory. /// /// Matched on the package name, not on "the directory is non-empty". A user with /// somebody else's pi extension installed was otherwise told their *Relay* /// extension was fine -- and, worse, a project-scoped install of an unrelated /// package raised a Relay trust warning about a file that has nothing to do with /// Relay. -fn relay_entry_in_directory(dir: &Path) -> Option { - std::fs::read_dir(dir) - .ok()? +/// +/// **Every** copy, not the first. pi walks the whole directory and resolves each +/// subdirectory's entry points independently (`collectAutoExtensionEntries`, pi +/// `v0.84.0`, `core/package-manager.ts:560`), so two copies here are two packages +/// to pi and both register hooks. Stopping at the first hid exactly the doubled +/// trace the duplicate check exists to refuse. +/// +/// Only the shapes pi accepts as an entry are considered -- a directory, or a +/// `.ts`/`.js` file. A flat copy puts a `package.json` beside `index.ts`, and that +/// manifest makes its own sibling look like this extension, so handing that file to +/// `-e` would give pi something it cannot import. Sorted because `read_dir` order is +/// undefined and the launcher's choice of copy must not vary run to run. +fn relay_entries_in_directory(dir: &Path) -> Vec { + let Ok(read) = std::fs::read_dir(dir) else { + return Vec::new(); + }; + let mut found: Vec = read .flatten() .map(|entry| entry.path()) - .find(|path| is_relay_extension(path)) + .filter(|path| { + let loadable = path.is_dir() + || matches!( + path.extension().and_then(std::ffi::OsStr::to_str), + Some("ts" | "js") + ); + loadable && is_relay_extension(path) + }) + .collect(); + found.sort(); + found } /// Whether a path is this extension: a package directory whose manifest names it, @@ -351,27 +392,42 @@ struct RecordedInstall { /// here -- an npm or git source is a name, not a location -- so those fall back to /// matching the package name inside the specifier, which is the best signal /// available without fetching anything. -fn relay_package_in_settings(settings: &Path) -> Option { - let base = settings.parent()?; - let raw = std::fs::read_to_string(settings).ok()?; - let value: serde_json::Value = serde_json::from_str(&raw).ok()?; - value.get("packages")?.as_array()?.iter().find_map(|entry| { - let source = package_source(entry)?; - let disabled = entry_disables_extensions(entry); - let resolved = base.join(source); - if is_relay_extension(&resolved) { - return Some(RecordedInstall { - path: resolved, - disabled, - }); - } - source - .contains(RELAY_PACKAGE_NAME) - .then(|| RecordedInstall { - path: PathBuf::from(source), - disabled, - }) - }) +/// Every entry, not the first: two entries naming different directories are two +/// identities to pi (`getPackageIdentity`, `core/package-manager.ts:1660`), so +/// `dedupePackages` keeps both and both load. +fn relay_packages_in_settings(settings: &Path) -> Vec { + let Some(base) = settings.parent() else { + return Vec::new(); + }; + let Some(value) = std::fs::read_to_string(settings) + .ok() + .and_then(|raw| serde_json::from_str::(&raw).ok()) + else { + return Vec::new(); + }; + let Some(entries) = value.get("packages").and_then(serde_json::Value::as_array) else { + return Vec::new(); + }; + entries + .iter() + .filter_map(|entry| { + let source = package_source(entry)?; + let resolved = base.join(source); + if is_relay_extension(&resolved) { + let disabled = entry_disables_extensions(entry, &resolved); + return Some(RecordedInstall { + path: resolved, + disabled, + }); + } + source + .contains(RELAY_PACKAGE_NAME) + .then(|| RecordedInstall { + path: PathBuf::from(source), + disabled: entry_disables_extensions(entry, Path::new(source)), + }) + }) + .collect() } /// The source string of one `packages` entry, whichever shape it was written in. @@ -383,26 +439,115 @@ fn package_source(entry: &serde_json::Value) -> Option<&str> { /// Whether an object-form entry's filters switch that package's extensions off. /// -/// Only the two shapes pi decides without consulting the package manifest are -/// recognized: an empty `extensions` array disables every extension file in the -/// package (`applyPackageFilter`, pi `v0.84.0`, `core/package-manager.ts:2208`), -/// and `autoload: false` starts from nothing, so an entry adding no `extensions` -/// patterns adds nothing back (`applyPackageDeltaFilter`, `:2232`). +/// An empty `extensions` array disables every extension file in the package +/// (`applyPackageFilter`, pi `v0.84.0`, `core/package-manager.ts:2208`), and +/// `autoload: false` starts from nothing, so an entry adding no patterns adds +/// nothing back (`applyPackageDeltaFilter`, `:2232`). +/// +/// **A non-empty list can disable just as completely, and that is the shape users +/// actually get.** pi's configuration selector switches one resource off by +/// appending `-` +/// (`interactive/components/config-selector.ts:607`), and a `-` pattern is pi's +/// last step, overriding every include (`applyPatterns`, `:745-755`). Reading a +/// non-empty list as "enabled" reported a plain Pass for a package pi loads +/// nothing from -- the same silent drop the rest of this module exists to catch. /// -/// A non-empty pattern list is matched against the manifest, which this module does -/// not read, so it counts as enabled. Guessing wrong in that direction produces the -/// false negative this module exists to prevent. -fn entry_disables_extensions(entry: &serde_json::Value) -> bool { +/// Deciding that needs the entry points the manifest declares, because that is +/// what pi matches the patterns against. Reading them is not the entry-point +/// precedence `-e` leaves to pi: the launcher still hands pi the directory and +/// lets pi choose what to load from it. +fn entry_disables_extensions(entry: &serde_json::Value, path: &Path) -> bool { let Some(object) = entry.as_object() else { return false; }; - match object + let autoload_off = object.get("autoload") == Some(&serde_json::Value::Bool(false)); + let Some(patterns) = object .get("extensions") .and_then(serde_json::Value::as_array) + else { + return autoload_off; + }; + let patterns: Vec<&str> = patterns + .iter() + .filter_map(serde_json::Value::as_str) + .collect(); + if patterns.is_empty() { + return true; + } + let declared = manifest_extension_entries(path); + // Nothing to compare against -- a source that is not on disk, or a manifest whose + // entries are globs pi expands with `minimatch`. pi has the files and this does + // not, so the answer that does not warn is the honest one: a missing warning costs + // a user a puzzle, a false one costs them trust in the whole check. + !declared.is_empty() + && declared + .iter() + .all(|entry| force_excluded(entry, &patterns, autoload_off)) +} + +/// Whether a filter's patterns leave one declared entry point switched off. +/// +/// A force-exclude is pi's final step and overrides every include, so under a normal +/// filter any `-` naming the file settles it (`applyPatterns`, pi `v0.84.0`, +/// `core/package-manager.ts:752`). Under an `autoload: false` delta the patterns are +/// replayed in order and the last one naming the file wins instead +/// (`applyAutoloadDisabledPatterns`, `:760`); a file no pattern names is never added +/// back, which the caller's empty-list branch already covers. +fn force_excluded(entry: &str, patterns: &[&str], autoload_off: bool) -> bool { + let mut naming = patterns + .iter() + .filter_map(|pattern| exact_pattern(pattern)) + .filter(|(_, target)| *target == entry); + if autoload_off { + return naming.next_back().is_some_and(|(marker, _)| marker == '-'); + } + naming.any(|(marker, _)| marker == '-') +} + +/// The `+`/`-` marker and the path an exact pattern names, when it is one. +/// +/// pi compares these as strings rather than globs, after stripping a leading `./` +/// (`normalizeExactPattern`, pi `v0.84.0`, `core/package-manager.ts:656`), which is +/// what makes them decidable from here at all. +fn exact_pattern(pattern: &str) -> Option<(char, &str)> { + let marker = pattern.chars().next()?; + if marker != '+' && marker != '-' { + return None; + } + let target = &pattern[marker.len_utf8()..]; + Some((marker, target.strip_prefix("./").unwrap_or(target))) +} + +/// The extension entry points a package declares, relative to its own root. +/// +/// Empty when they cannot be pinned down: a source that is not on disk, or a +/// manifest entry carrying a glob or an override marker, which pi expands with +/// `globSync` and `minimatch` rather than comparing as a string. The caller reads +/// empty as "cannot tell" and does not warn. +fn manifest_extension_entries(path: &Path) -> Vec { + let Some(root) = package_root(path) else { + return Vec::new(); + }; + let entries: Vec = std::fs::read_to_string(root.join("package.json")) + .ok() + .and_then(|raw| serde_json::from_str::(&raw).ok()) + .as_ref() + .and_then(|manifest| manifest.get("pi")?.get("extensions")?.as_array()) + .map(|entries| { + entries + .iter() + .filter_map(serde_json::Value::as_str) + .map(|entry| entry.strip_prefix("./").unwrap_or(entry).to_string()) + .collect() + }) + .unwrap_or_default(); + if entries + .iter() + .any(|entry| entry.contains(['*', '?']) || entry.starts_with(['+', '-', '!'])) { - Some(patterns) => patterns.is_empty(), - None => object.get("autoload") == Some(&serde_json::Value::Bool(false)), + return Vec::new(); } + entries } /// `/settings.json`, where `pi install` records a user-scope package. diff --git a/crates/cli/src/agents/pi/launch.rs b/crates/cli/src/agents/pi/launch.rs index 8ce0e682d..cda9e2af7 100644 --- a/crates/cli/src/agents/pi/launch.rs +++ b/crates/cli/src/agents/pi/launch.rs @@ -86,6 +86,24 @@ pub(crate) fn prepare( duplicate.display() ))); } + // A project-scoped copy is the one duplicate that cannot be decided from here. pi loads + // `/.pi/extensions` only for a trusted project, and that decision is made inside the + // run -- `-a` overrides it, `defaultProjectTrust` can pre-answer it, and "trust this session + // only" persists nothing -- so refusing would block every launch in an untrusted project + // over a copy that will not load. Say what happens if the project *is* trusted instead. + for site in super::doctor::relay_extension_sites(¤t_dir()) + .iter() + .filter(|site| site.scope == super::doctor::ExtensionScope::Project) + { + launch.notes.push(format!( + "{} is a second copy of this extension under the project's `.pi/`. pi loads it \ + beside the one passed with `-e` whenever the project is trusted, and then every \ + turn, tool and inline-shell event is reported twice. Remove it, or run in an \ + untrusted project", + site.path.display() + )); + } + let rendered = path.display().to_string(); set_env(launch, PI_EXTENSION_PATH_ENV, &rendered); insert_after_host( diff --git a/crates/cli/src/diagnostics/mod.rs b/crates/cli/src/diagnostics/mod.rs index 33f124083..58dc840de 100644 --- a/crates/cli/src/diagnostics/mod.rs +++ b/crates/cli/src/diagnostics/mod.rs @@ -524,6 +524,26 @@ fn pi_extension_trust_check(cwd: &Path) -> Check { .filter(|site| site.scope == crate::agents::pi::doctor::ExtensionScope::Project) .collect(); + // Ahead of the project warning, because a project copy must not hide it: the two ungated + // copies load whether or not the project is trusted, and that is the louder problem. + if let Some(site) = sites.first() + && let Some(duplicate) = + crate::agents::pi::doctor::conflicting_extension_site(cwd, &site.path) + { + return Check { + name: NAME, + status: Status::Warn, + details: format!( + "two copies would load: {} and {}. pi de-duplicates by path, not by package, \ + so both register hooks and every event is reported twice -- each turn is \ + closed as superseded by its own duplicate, and the inline-shell gate decides \ + one command twice. Keep one copy", + site.path.display(), + duplicate.display() + ), + }; + } + if let Some(project) = project_sites.first() { return Check { name: NAME, @@ -543,23 +563,6 @@ fn pi_extension_trust_check(cwd: &Path) -> Check { } match sites.first() { - // Reachable without the launcher at all: pi scans its extensions directory and its - // recorded packages independently, so a user holding both a copy and an install - // double-loads under plain `pi`, with no Relay command involved. - Some(site) - if let Some(duplicate) = - crate::agents::pi::doctor::conflicting_extension_site(cwd, &site.path) => - { - Check { - name: NAME, - status: Status::Warn, - details: format!( - "two copies would load: {} and {}. pi de-duplicates by path, not by package, so both register hooks and every event is reported twice -- each turn is closed as superseded by its own duplicate, and the inline-shell gate decides one command twice. Keep one copy", - site.path.display(), - duplicate.display() - ), - } - } // Installed, and switched off in pi's own settings. Same silent drop as the trust gate, // from the other direction -- and `-e` ignores those filters, so the launcher still // instruments a session the user's own `pi` runs are missing. @@ -567,7 +570,10 @@ fn pi_extension_trust_check(cwd: &Path) -> Check { name: NAME, status: Status::Warn, details: format!( - "{} is recorded in pi's settings with its extensions filtered off, so pi does not load it -- remove the `extensions` filter, or the `autoload: false`, on that entry. `nemo-relay run --agent pi` is unaffected: it passes `-e`, which applies no settings filter", + "{} is recorded in pi's settings with its extensions filtered off, so pi does not \ + load it -- remove the `extensions` filter, or the `autoload: false`, on \ + that entry. `nemo-relay run --agent pi` is unaffected: it passes `-e`, \ + which applies no settings filter", site.path.display() ), }, diff --git a/crates/cli/src/sessions/idle.rs b/crates/cli/src/sessions/idle.rs index 7833ef124..76d1aef90 100644 --- a/crates/cli/src/sessions/idle.rs +++ b/crates/cli/src/sessions/idle.rs @@ -90,7 +90,9 @@ async fn close_idle_turns(idle_sessions: Vec<(String, Session)>, reason: &str) - for (session_id, mut session) in idle_sessions { let stack = session.scope_stack.clone(); match TASK_SCOPE_STACK - .scope(stack, async { session.close_turn_for_reason(reason).await }) + .scope(stack, async { + session.close_idle_scopes_for_reason(reason).await + }) .await { Ok((subagent_ids, subscriber_delivery)) => { diff --git a/crates/cli/src/sessions/mod.rs b/crates/cli/src/sessions/mod.rs index a45c4a24c..f491a1360 100644 --- a/crates/cli/src/sessions/mod.rs +++ b/crates/cli/src/sessions/mod.rs @@ -799,7 +799,7 @@ impl Session { } fn is_idle_for(&self, now: Instant, timeout: Duration) -> bool { - self.turn_scope.is_some() + (self.turn_scope.is_some() || self.holds_only_an_unannounced_agent_scope()) && self.active_gateway_calls == 0 && self.llms.is_empty() && self.tools.is_empty() @@ -1265,13 +1265,50 @@ impl Session { } let (_, turn_delivery) = self.close_turn_for_reason("closed_by_agent_end").await?; self.clear_correlation_state(); - let agent_delivery = self.close_agent_scope(event.payload)?; + let agent_delivery = self.close_agent_scope(event.payload, Some(event.metadata))?; self.session_started = false; // Agent end is queued after turn end on the serial dispatcher. Waiting for the later // receipt therefore covers both terminal events without a process-wide flush. Ok(agent_delivery.or(turn_delivery)) } + // Closes what the idle sweeper found, which is normally the open turn. + // + // A session no harness lifecycle event ever announced has no turn to close: for a harness + // with an explicit turn start, a mark opens only the agent scope, so closing the turn alone + // would leave that scope -- and the session holding it -- resident until process shutdown. + // Sessions the harness *did* announce are left alone, because sitting at an idle prompt + // between turns is normal and closing the session there would split one run into two traces. + async fn close_idle_scopes_for_reason( + &mut self, + reason: &str, + ) -> Result<(Vec, Option), CliError> { + let (closed_subagents, turn_delivery) = self.close_turn_for_reason(reason).await?; + if !self.holds_only_an_unannounced_agent_scope() { + return Ok((closed_subagents, turn_delivery)); + } + let agent_delivery = self.close_agent_scope(json!({ "status": reason }), None)?; + Ok((closed_subagents, agent_delivery.or(turn_delivery))) + } + + // Whether this session's only content is an agent scope no harness lifecycle event asked for. + // + // `turn_index` stays at zero until the first turn opens, so a session that did real work + // keeps its agent scope even when its session-start hook was lost -- without that guard, a + // later `turn_start` would open a second agent scope on the same stack and split the trace. + fn holds_only_an_unannounced_agent_scope(&self) -> bool { + !self.session_started + && self.turn_index == 0 + && self.agent_scope.is_some() + && self.turn_scope.is_none() + && self.subagents.is_empty() + && self.subagent_stacks.is_empty() + && self.subagent_stack.is_empty() + && self.llms.is_empty() + && self.tools.is_empty() + && self.active_gateway_calls == 0 + } + async fn close_for_shutdown(&mut self, reason: &str) -> Result<(), CliError> { let stack = self.scope_stack.clone(); let payload = json!({ "status": reason }); @@ -1282,7 +1319,7 @@ impl Session { } let _ = self.close_turn_for_reason(reason).await?; self.clear_correlation_state(); - let _ = self.close_agent_scope(payload)?; + let _ = self.close_agent_scope(payload, None)?; self.session_started = false; Ok(()) }) @@ -1352,9 +1389,15 @@ impl Session { // Ends the root agent scope when present. Duplicate agent-end hooks can reach this path after the // scope is already gone, so absence is treated as a no-op. + // Takes the closing hook's metadata for the same reason `close_turn_scope` does: a scope end + // otherwise repeats the metadata its scope was *opened* with, so pi's session end named + // `session_start` as the hook that produced it and bucketing ATOF by `hook_event_name` never + // yielded a session end. Synthetic closes pass `None` and keep the opening identity, because + // no hook stands behind them. fn close_agent_scope( &mut self, payload: Value, + boundary_metadata: Option, ) -> Result, CliError> { let Some(scope) = self.agent_scope.take() else { return Ok(None); @@ -1363,6 +1406,7 @@ impl Session { PopScopeParams::builder() .handle_uuid(&scope.uuid) .output(payload) + .metadata_opt(boundary_metadata) .build(), )?; Ok(Some(subscriber_delivery)) diff --git a/crates/cli/tests/coverage/agents/pi_doctor_tests.rs b/crates/cli/tests/coverage/agents/pi_doctor_tests.rs index b4c537da0..5e41b0336 100644 --- a/crates/cli/tests/coverage/agents/pi_doctor_tests.rs +++ b/crates/cli/tests/coverage/agents/pi_doctor_tests.rs @@ -11,7 +11,13 @@ use crate::test_support::EnvScope; /// A directory pi would see as this extension: a package manifest naming it. fn write_relay_package(dir: &std::path::Path) { std::fs::create_dir_all(dir).unwrap(); - std::fs::write(dir.join("package.json"), r#"{"name": "nemo-relay-pi"}"#).unwrap(); + // The declared entry point matters: it is what a `packages` filter's patterns are matched + // against, so a fixture without one reads as "cannot tell" rather than as enabled. + std::fs::write( + dir.join("package.json"), + r#"{"name": "nemo-relay-pi", "pi": {"extensions": ["./index.ts"]}}"#, + ) + .unwrap(); std::fs::write(dir.join("index.ts"), "export default 1").unwrap(); } @@ -363,6 +369,96 @@ fn an_object_form_package_entry_is_found() { // A non-empty pattern list is matched against the package manifest, which this // module does not read. Reporting it as loaded is the deliberate direction: a false // negative here is the exact failure the whole module exists to prevent. +// The other side of the same table. A `+` include, a bare name, and anything this module +// cannot decide -- a glob, which pi expands with `minimatch` -- all count as enabled, because +// a false warning costs more than a missing one. +#[test] +fn an_object_form_entry_whose_patterns_leave_it_loaded_is_not_reported_as_disabled() { + for body in [ + r#"{"packages": [{"source": "../checkout", "extensions": ["+index.ts"]}]}"#, + r#"{"packages": [{"source": "../checkout", "extensions": ["index.ts"]}]}"#, + r#"{"packages": [{"source": "../checkout", "autoload": false, "extensions": ["+index.ts"]}]}"#, + // Deliberate fail-open: pi's glob matcher is not reimplemented here. + r#"{"packages": [{"source": "../checkout", "extensions": ["!*.ts"]}]}"#, + ] { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("checkout")); + std::fs::write(agent_dir.join("settings.json"), body).unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + assert_eq!(sites.len(), 1, "{body}: {sites:?}"); + assert!(!sites[0].disabled_by_settings, "{body}"); + } +} + +// B1: two copies inside ONE source. pi resolves every distinct package source, so both load +// and post every hook twice -- and stopping at the first match hid exactly that. +#[test] +fn two_copies_recorded_in_one_settings_file_are_both_reported() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("one")); + write_relay_package(&temp.path().join("two")); + std::fs::write( + agent_dir.join("settings.json"), + r#"{"packages": ["../one", "../two"]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + assert_eq!(sites.len(), 2, "{sites:?}"); + let launched = launchable_extension_path(temp.path()).unwrap(); + assert!(conflicting_extension_site(temp.path(), &launched).is_some()); +} + +#[test] +fn two_copies_in_one_extensions_directory_are_both_reported() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + let extensions = agent_dir.join("extensions"); + write_relay_package(&extensions.join("nemo-relay")); + write_relay_package(&extensions.join("nemo-relay-copy")); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + assert_eq!(sites.len(), 2, "{sites:?}"); + let launched = launchable_extension_path(temp.path()).unwrap(); + assert!(conflicting_extension_site(temp.path(), &launched).is_some()); +} + +// B2: a copy pi's own settings switch off is reliably *not* a load, so counting it refused a +// launch over a copy that was never going to register a hook. +#[test] +fn a_copy_pi_switched_off_is_not_a_second_copy() { + for body in [ + r#"{"packages": [{"source": "../off", "extensions": []}]}"#, + r#"{"packages": [{"source": "../off", "autoload": false}]}"#, + ] { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("off")); + let explicit = temp.path().join("checkout"); + write_relay_package(&explicit); + std::fs::write(agent_dir.join("settings.json"), body).unwrap(); + + let _env = scoped(Some(explicit.as_os_str()), Some(agent_dir.as_os_str())); + assert_eq!( + conflicting_extension_site(temp.path(), &explicit), + None, + "{body}" + ); + } +} + #[test] fn an_object_form_entry_with_extension_patterns_is_still_found() { let temp = tempfile::tempdir().unwrap(); @@ -390,6 +486,12 @@ fn an_object_form_entry_whose_extensions_are_disabled_is_reported_as_disabled() for body in [ r#"{"packages": [{"source": "../checkout", "extensions": []}]}"#, r#"{"packages": [{"source": "../checkout", "autoload": false}]}"#, + // What pi's own configuration selector writes when a user switches this extension off: + // `-`, a force-exclude pi applies last and unconditionally. Reading a non-empty + // list as "enabled" reported a plain Pass for a package pi loads nothing from. + r#"{"packages": [{"source": "../checkout", "extensions": ["-index.ts"]}]}"#, + r#"{"packages": [{"source": "../checkout", "extensions": ["-./index.ts"]}]}"#, + r#"{"packages": [{"source": "../checkout", "autoload": false, "extensions": ["-index.ts"]}]}"#, ] { let temp = tempfile::tempdir().unwrap(); let agent_dir = temp.path().join("agent"); diff --git a/crates/cli/tests/coverage/shared/session_tests.rs b/crates/cli/tests/coverage/shared/session_tests.rs index 7a776da30..ebd971913 100644 --- a/crates/cli/tests/coverage/shared/session_tests.rs +++ b/crates/cli/tests/coverage/shared/session_tests.rs @@ -5566,10 +5566,97 @@ async fn pi_turn_scopes_open_at_pis_own_boundary_and_carry_attempt_attribution() 1, "the one turn should close exactly once" ); + + // Every scope end names its own closer, not the hook that opened the scope. The session end + // reported `session_start` while `close_agent_scope` had no metadata channel, so bucketing + // ATOF by `hook_event_name` never yielded a session end at all. + let scope_end = |name: &str| { + captured + .iter() + .find(|(candidate, category, _)| { + candidate == name && *category == Some(ScopeCategory::End) + }) + .unwrap_or_else(|| panic!("{name} should have closed")) + .2 + .clone() + }; + assert_eq!( + scope_end("pi")["hook_event_name"], + json!("session_shutdown") + ); + assert_eq!(scope_end("pi-turn")["hook_event_name"], json!("turn_end")); + assert_eq!( + scope_end("read")["hook_event_name"], + json!("tool_execution_end") + ); + // The opening identity is untouched, so a start and its end remain distinguishable. + let session_start = captured + .iter() + .find(|(name, category, _)| name == "pi" && *category == Some(ScopeCategory::Start)) + .expect("the session scope should exist"); + assert_eq!(session_start.2["hook_event_name"], json!("session_start")); + drop(captured); deregister_subscriber(subscriber_name).unwrap(); } +// A pi hook that resolves to nothing -- an empty object, an unknown name -- still opens a +// session, under a synthetic id no later `session_shutdown` can ever match. pi's marks land on +// the session scope rather than a turn, which is what `has_explicit_turn_start` is for, and the +// idle sweeper's precondition was an open turn -- so those sessions were resident until process +// shutdown while Codex's and Claude Code's were swept. +#[tokio::test] +async fn pi_sessions_that_only_ever_held_a_mark_are_swept() { + let manager = SessionManager::new(session_test_config()); + for index in 0..3 { + apply_pi_hook( + &manager, + json!({ "session_id": format!("pi-sparse-{index}") }), + ) + .await; + } + assert_eq!(manager.inner.lock().await.len(), 3); + + let closed = manager + .close_idle_sessions_at(Instant::now(), Duration::from_secs(0), "idle_timeout") + .await + .unwrap(); + + assert_eq!(closed, 3, "every unannounced session should be swept"); + assert_eq!(manager.inner.lock().await.len(), 0); +} + +// The guard that keeps the sweep narrow. A session pi announced is a user sitting at an idle +// prompt between turns, and closing it there would split one run into two traces. +#[tokio::test] +async fn an_announced_pi_session_idling_between_turns_is_not_swept() { + let manager = SessionManager::new(session_test_config()); + let session = json!({ "session_id": "pi-announced-session" }); + for payload in [ + json!({ "hook_event_name": "session_start", "reason": "startup" }), + json!({ "hook_event_name": "turn_start", "turn_index": 0, "turn_seq": 0 }), + json!({ "hook_event_name": "turn_end", "turn_index": 0, "turn_seq": 0 }), + ] { + let mut merged = session.clone(); + merged + .as_object_mut() + .unwrap() + .extend(payload.as_object().unwrap().clone()); + apply_pi_hook(&manager, merged).await; + } + + let closed = manager + .close_idle_sessions_at(Instant::now(), Duration::from_secs(0), "idle_timeout") + .await + .unwrap(); + + assert_eq!( + closed, 0, + "an announced session between turns is not idle debris" + ); + assert_eq!(manager.inner.lock().await.len(), 1); +} + // The re-entry case, which is why `turn_seq` exists: pi's `turn_index` restarts at 0 on every // attempt, so two turns in one session both call themselves turn 0. #[tokio::test] From 5616c3591e3c8ecad44b7dd509ed98529119567b Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 14:33:54 -0700 Subject: [PATCH 35/41] chore(pi): remove four stray files committed by accident `0"`, `12"`, `102"` and `1206"` are shell-redirect debris -- a misquoted `grep -c` wrote its target into a file named after the count. Each holds one line of `target/debug/deps/...` build output under an absolute local path. They were swept in by a `git add -A` in `01411c19`. They had been sitting untracked in the tree, were noticed, and were judged to predate the branch and left alone -- and then committed by the next blanket add, which is exactly the gap between "not mine" and "not staged". Removed rather than rewritten: they are already pushed, so an amend would not unpublish the path, and rewriting published commits on a branch under review is the manoeuvre that previously landed eight unsigned commits here. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- "0\"" | 1 - "102\"" | 1 - "12\"" | 1 - "1206\"" | 1 - 4 files changed, 4 deletions(-) delete mode 100644 "0\"" delete mode 100644 "102\"" delete mode 100644 "12\"" delete mode 100644 "1206\"" diff --git "a/0\"" "b/0\"" deleted file mode 100644 index edea114c6..000000000 --- "a/0\"" +++ /dev/null @@ -1 +0,0 @@ -"/Users/yuchenz/Desktop/Work/Project/NeMo-Relay/target/debug/deps/nemo_relay-05557a0b1c85a83a - diff --git "a/102\"" "b/102\"" deleted file mode 100644 index c490ea491..000000000 --- "a/102\"" +++ /dev/null @@ -1 +0,0 @@ -"/Users/yuchenz/Desktop/Work/Project/NeMo-Relay/target/debug/deps/cli_tests-2f35b16f0798cdf9 - diff --git "a/12\"" "b/12\"" deleted file mode 100644 index ea9391eea..000000000 --- "a/12\"" +++ /dev/null @@ -1 +0,0 @@ -"/Users/yuchenz/Desktop/Work/Project/NeMo-Relay/target/debug/deps/architecture_tests-2096cbd4d612e5af - diff --git "a/1206\"" "b/1206\"" deleted file mode 100644 index 771012a08..000000000 --- "a/1206\"" +++ /dev/null @@ -1 +0,0 @@ -"/Users/yuchenz/Desktop/Work/Project/NeMo-Relay/target/debug/deps/nemo_relay_cli-40a232977d19ebc8 - From 2063ed874b268b45a9274fe130a995e7e5c0857f Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 14:42:47 -0700 Subject: [PATCH 36/41] fix(pi): choose the copy pi already loads, and name the fault's origin Five findings, four of them on the discovery code the last round added. **Selecting the first site could manufacture the duplicate it then refused.** `launchable_extension_path` took the first ungated site regardless of its filter, so with a disabled copy recorded before an enabled one it chose the disabled one -- and `-e` applies no settings filter, so that choice *re-enabled* it and the enabled copy became a genuine second load. The launch was refused over a duplicate the choice created. It prefers a copy pi already loads now, and falls back to a filtered-off one only when that is all there is, because `-e` still makes that work. The doctor's duplicate check asks the same question of the same copy rather than of whichever site sorts first. **`disabled_by_settings` was a bool, and the third state was the common one.** pi sorts patterns into force-include, force-exclude, exclude and include, and only the first two are exact strings; the rest are globs it matches with `minimatch`. An include list that never names our entry (`["other.ts"]`) or an `autoload: false` delta that adds something else back are both decidable and were reported as loaded -- and a genuine glob was reported as loaded too, which is the claim this module exists to stop making on no evidence. The verdict is now `Loads` / `Excluded` / `Undecided`, and doctor warns on the third rather than passing. **The project-copy launch note described copies that cannot double a trace.** It included entries pi's settings switch off, and entries canonically identical to the launched package, which pi de-duplicates by path. **`reached: false` still conflated three faults.** A timeout is not an unreachable gateway -- it may be up and slow, and posts are serialized so a gate also waits out its queue -- and a handler failure is not a transport result at all. The fault carries a four-way origin now (`transport`, `timeout`, `response`, `handler`) with one opening each and the same tail, since nothing judged the request in any of them. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- crates/cli/src/agents/pi/doctor.rs | 302 +++++++++++++----- crates/cli/src/agents/pi/launch.rs | 7 +- crates/cli/src/diagnostics/mod.rs | 32 +- .../tests/coverage/agents/launcher_tests.rs | 51 +++ .../tests/coverage/agents/pi_doctor_tests.rs | 83 ++++- docs/nemo-relay-cli/pi.mdx | 8 +- integrations/pi/README.md | 6 +- integrations/pi/index.ts | 6 +- integrations/pi/src/gateway-client.ts | 64 ++-- integrations/pi/test/gateway-client.test.mjs | 36 ++- 10 files changed, 456 insertions(+), 139 deletions(-) diff --git a/crates/cli/src/agents/pi/doctor.rs b/crates/cli/src/agents/pi/doctor.rs index cce061ac4..4d7d14750 100644 --- a/crates/cli/src/agents/pi/doctor.rs +++ b/crates/cli/src/agents/pi/doctor.rs @@ -72,18 +72,22 @@ impl ExtensionScope { pub(crate) struct ExtensionSite { pub(crate) path: PathBuf, pub(crate) scope: ExtensionScope, - /// Whether pi's own settings switch this copy off. + /// What pi's own settings do to this copy. /// - /// An object-form `packages` entry carries per-resource filters, and two shapes leave nothing - /// enabled for extensions: an empty `extensions` array, and `autoload: false` with no - /// extension patterns (pi `v0.84.0`, `core/package-manager.ts:2208` and `:2232`). The copy is - /// installed and pi still never loads it, which is the same silent drop as the trust gate and - /// must not read as a plain Pass. + /// A copy pi will not load is installed and still silently absent -- the same failure as the + /// trust gate, from another direction -- so it must not read as a plain Pass. A copy whose + /// filter this check cannot evaluate must not read as one either. /// - /// A launch is unaffected on purpose: `-e` resolves its argument with no filter at all, so - /// `launchable_extension_path` ignores this flag. The user asked for instrumentation by - /// running the launcher; the warning is for their own `pi` sessions. - pub(crate) disabled_by_settings: bool, + /// `-e` applies no settings filter, so a launch can still use an `Excluded` copy; that is + /// deliberate, and it is why `launchable_extension_path` merely *prefers* a loading one. + pub(crate) filter: SettingsFilter, +} + +impl ExtensionSite { + /// Whether pi loads this copy without help -- what makes it a second copy beside `-e`. + fn loads_on_its_own(&self) -> bool { + self.scope != ExtensionScope::Project && self.filter == SettingsFilter::Loads + } } /// Human-readable hook status for `nemo-relay doctor`. @@ -176,7 +180,7 @@ pub(crate) fn relay_extension_sites(cwd: &Path) -> Vec { sites.push(ExtensionSite { path, scope: ExtensionScope::Explicit, - disabled_by_settings: false, + filter: SettingsFilter::Loads, }); } let discovered = |sites: &mut Vec, dir: &Path, scope| { @@ -186,7 +190,7 @@ pub(crate) fn relay_extension_sites(cwd: &Path) -> Vec { .map(|path| ExtensionSite { path, scope, - disabled_by_settings: false, + filter: SettingsFilter::Loads, }), ); }; @@ -197,7 +201,7 @@ pub(crate) fn relay_extension_sites(cwd: &Path) -> Vec { .map(|install| ExtensionSite { path: install.path, scope, - disabled_by_settings: install.disabled, + filter: install.filter, }), ); }; @@ -240,11 +244,24 @@ pub(crate) fn relay_extension_sites(cwd: &Path) -> Vec { /// entry-point precedence is pi's to change. A copy pi would *not* have /// discovered is a different matter, and is what `conflicting_extension_site` /// is for. +/// A copy pi already loads is preferred over one its settings switch off, even +/// when the disabled one comes first. `-e` applies no settings filter, so passing +/// the disabled copy would *re-enable* it -- and then the enabled one is a genuine +/// second copy and the launch is refused over a duplicate the choice manufactured. +/// Choosing the enabled copy leaves the disabled one disabled and loads exactly +/// once. A disabled copy is still used when it is all there is, because `-e` makes +/// it work. pub(crate) fn launchable_extension_path(cwd: &Path) -> Option { - relay_extension_sites(cwd) - .into_iter() - .find(|site| site.scope != ExtensionScope::Project && site.path.exists()) - .map(|site| site.path) + let sites = relay_extension_sites(cwd); + let usable = || { + sites + .iter() + .filter(|site| site.scope != ExtensionScope::Project && site.path.exists()) + }; + usable() + .find(|site| site.filter == SettingsFilter::Loads) + .or_else(|| usable().next()) + .map(|site| site.path.clone()) } /// A *second* copy of this extension that pi would load beside the launched one. @@ -288,11 +305,32 @@ pub(crate) fn conflicting_extension_site(cwd: &Path, launched: &Path) -> Option< let launched_root = package_root(launched); relay_extension_sites(cwd) .into_iter() - .filter(|site| site.scope != ExtensionScope::Project && !site.disabled_by_settings) + .filter(ExtensionSite::loads_on_its_own) .find(|site| package_root(&site.path) != launched_root) .map(|site| site.path) } +/// Project-scoped copies that would load *beside* the launched one, if the project +/// is trusted. +/// +/// Two exclusions, both for the same reason the launcher does not refuse on these: +/// a copy pi's settings switch off never loads, and a copy that canonicalizes to +/// the launched package is the same package -- pi de-duplicates by path, so it +/// loads once. Warning about either would describe a doubled trace that cannot +/// happen. +pub(crate) fn project_copies_beside(cwd: &Path, launched: &Path) -> Vec { + let launched_root = package_root(launched); + relay_extension_sites(cwd) + .into_iter() + .filter(|site| { + site.scope == ExtensionScope::Project + && site.filter != SettingsFilter::Excluded + && package_root(&site.path) != launched_root + }) + .map(|site| site.path) + .collect() +} + /// The package directory a site belongs to, or `None` when it is not on disk. /// /// A source that is not a path -- an npm or git specifier `pi install` recorded -- @@ -371,7 +409,7 @@ fn manifest_names_relay(manifest: &Path) -> bool { /// A `packages` entry that records this extension, and whether pi will load it. struct RecordedInstall { path: PathBuf, - disabled: bool, + filter: SettingsFilter, } /// The NeMo Relay package among the sources `pi install` recorded, if any. @@ -414,17 +452,17 @@ fn relay_packages_in_settings(settings: &Path) -> Vec { let source = package_source(entry)?; let resolved = base.join(source); if is_relay_extension(&resolved) { - let disabled = entry_disables_extensions(entry, &resolved); + let filter = entry_filters_extensions(entry, &resolved); return Some(RecordedInstall { path: resolved, - disabled, + filter, }); } source .contains(RELAY_PACKAGE_NAME) .then(|| RecordedInstall { path: PathBuf::from(source), - disabled: entry_disables_extensions(entry, Path::new(source)), + filter: entry_filters_extensions(entry, Path::new(source)), }) }) .collect() @@ -437,93 +475,201 @@ fn package_source(entry: &serde_json::Value) -> Option<&str> { .or_else(|| entry.get("source").and_then(serde_json::Value::as_str)) } -/// Whether an object-form entry's filters switch that package's extensions off. -/// -/// An empty `extensions` array disables every extension file in the package -/// (`applyPackageFilter`, pi `v0.84.0`, `core/package-manager.ts:2208`), and -/// `autoload: false` starts from nothing, so an entry adding no patterns adds -/// nothing back (`applyPackageDeltaFilter`, `:2232`). -/// -/// **A non-empty list can disable just as completely, and that is the shape users -/// actually get.** pi's configuration selector switches one resource off by -/// appending `-` -/// (`interactive/components/config-selector.ts:607`), and a `-` pattern is pi's -/// last step, overriding every include (`applyPatterns`, `:745-755`). Reading a -/// non-empty list as "enabled" reported a plain Pass for a package pi loads -/// nothing from -- the same silent drop the rest of this module exists to catch. +/// What an object-form entry's filters do to that package's extensions. /// -/// Deciding that needs the entry points the manifest declares, because that is -/// what pi matches the patterns against. Reading them is not the entry-point -/// precedence `-e` leaves to pi: the launcher still hands pi the directory and -/// lets pi choose what to load from it. -fn entry_disables_extensions(entry: &serde_json::Value, path: &Path) -> bool { +/// pi sorts a pattern list into four buckets: `+x` force-include and `-x` +/// force-exclude, both compared as exact strings, and `!x` exclude and bare `x` +/// include, both matched as globs (`applyPatterns`, pi `v0.84.0`, +/// `core/package-manager.ts:712-756`). Only the exact ones can be decided from +/// here without reimplementing `minimatch`, which is why the answer is a tri-state +/// rather than a bool: reporting a package as loaded because a glob could not be +/// read is the same silent Pass this module exists to prevent. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SettingsFilter { + /// pi loads this package's extensions. + Loads, + /// pi loads none of them -- the entry point is filtered out for certain. + Excluded, + /// The filter turns on a glob this check does not evaluate. pi has the files + /// and the matcher; this does not, so it says so rather than guessing. + Undecided, +} + +fn entry_filters_extensions(entry: &serde_json::Value, path: &Path) -> SettingsFilter { let Some(object) = entry.as_object() else { - return false; + return SettingsFilter::Loads; }; let autoload_off = object.get("autoload") == Some(&serde_json::Value::Bool(false)); let Some(patterns) = object .get("extensions") .and_then(serde_json::Value::as_array) else { - return autoload_off; + // `autoload: false` with no patterns starts from nothing and adds nothing back + // (`applyPackageDeltaFilter`, `core/package-manager.ts:2232`). + return if autoload_off { + SettingsFilter::Excluded + } else { + SettingsFilter::Loads + }; }; let patterns: Vec<&str> = patterns .iter() .filter_map(serde_json::Value::as_str) .collect(); + // An empty array explicitly disables every resource of the type + // (`applyPackageFilter`, `core/package-manager.ts:2208`). if patterns.is_empty() { - return true; + return SettingsFilter::Excluded; } let declared = manifest_extension_entries(path); - // Nothing to compare against -- a source that is not on disk, or a manifest whose - // entries are globs pi expands with `minimatch`. pi has the files and this does - // not, so the answer that does not warn is the honest one: a missing warning costs - // a user a puzzle, a false one costs them trust in the whole check. - !declared.is_empty() - && declared - .iter() - .all(|entry| force_excluded(entry, &patterns, autoload_off)) + if declared.is_empty() { + return SettingsFilter::Undecided; + } + let verdicts = declared + .iter() + .map(|entry| entry_verdict(entry, &patterns, autoload_off)); + let mut excluded = true; + for verdict in verdicts { + match verdict { + SettingsFilter::Undecided => return SettingsFilter::Undecided, + SettingsFilter::Loads => excluded = false, + SettingsFilter::Excluded => {} + } + } + if excluded { + SettingsFilter::Excluded + } else { + SettingsFilter::Loads + } } -/// Whether a filter's patterns leave one declared entry point switched off. +/// What a filter's patterns do to one declared entry point. /// -/// A force-exclude is pi's final step and overrides every include, so under a normal -/// filter any `-` naming the file settles it (`applyPatterns`, pi `v0.84.0`, -/// `core/package-manager.ts:752`). Under an `autoload: false` delta the patterns are -/// replayed in order and the last one naming the file wins instead -/// (`applyAutoloadDisabledPatterns`, `:760`); a file no pattern names is never added -/// back, which the caller's empty-list branch already covers. -fn force_excluded(entry: &str, patterns: &[&str], autoload_off: bool) -> bool { - let mut naming = patterns - .iter() - .filter_map(|pattern| exact_pattern(pattern)) - .filter(|(_, target)| *target == entry); +/// Under a normal filter the order is include, exclude, force-include, +/// force-exclude, and only the last step is unconditional -- so a `-` naming the +/// entry settles it, and otherwise the answer turns on whether any include or +/// exclude glob could match. Under an `autoload: false` delta the patterns are +/// replayed in order and the last one naming the entry wins, with an entry no +/// pattern names never added at all (`applyAutoloadDisabledPatterns`, +/// `core/package-manager.ts:760-777`). +fn entry_verdict(entry: &str, patterns: &[&str], autoload_off: bool) -> SettingsFilter { if autoload_off { - return naming.next_back().is_some_and(|(marker, _)| marker == '-'); + let mut decided = SettingsFilter::Excluded; + for pattern in patterns { + match classify_pattern(pattern, entry) { + PatternEffect::Unknown => return SettingsFilter::Undecided, + PatternEffect::Names(enabled) => { + decided = if enabled { + SettingsFilter::Loads + } else { + SettingsFilter::Excluded + }; + } + PatternEffect::Silent => {} + } + } + return decided; + } + + let mut force_excluded = false; + let mut force_included = false; + let mut includes = 0_usize; + let mut included = false; + let mut excluded = false; + for pattern in patterns { + let (marker, target) = split_pattern(pattern); + let literal = literal_target(target); + match marker { + Some('+') => force_included |= target_matches(target, entry), + Some('-') => force_excluded |= target_matches(target, entry), + Some('!') => match literal { + Some(literal) => excluded |= literal == entry, + // A glob exclude could remove the entry, and only a force-include + // would bring it back. + None if !force_included => return SettingsFilter::Undecided, + None => {} + }, + _ => { + includes += 1; + match literal { + Some(literal) => included |= literal == entry, + None => return SettingsFilter::Undecided, + } + } + } + } + if force_excluded { + return SettingsFilter::Excluded; + } + if force_included { + return SettingsFilter::Loads; + } + // With no includes at all, step 1 keeps everything. + if (includes == 0 || included) && !excluded { + SettingsFilter::Loads + } else { + SettingsFilter::Excluded + } +} + +/// What one `autoload: false` delta pattern does to a given entry. +enum PatternEffect { + /// Names the entry, and either enables or disables it. + Names(bool), + /// Cannot say -- the pattern is a glob. + Unknown, + /// Names something else. + Silent, +} + +fn classify_pattern(pattern: &str, entry: &str) -> PatternEffect { + let (marker, target) = split_pattern(pattern); + let enabled = !matches!(marker, Some('-') | Some('!')); + if matches!(marker, Some('+') | Some('-')) { + return if target_matches(target, entry) { + PatternEffect::Names(enabled) + } else { + PatternEffect::Silent + }; + } + match literal_target(target) { + Some(literal) if literal == entry => PatternEffect::Names(enabled), + Some(_) => PatternEffect::Silent, + None => PatternEffect::Unknown, + } +} + +/// The `+`/`-`/`!` marker and the rest of a pattern. +fn split_pattern(pattern: &str) -> (Option, &str) { + match pattern.chars().next() { + Some(marker @ ('+' | '-' | '!')) => (Some(marker), &pattern[marker.len_utf8()..]), + _ => (None, pattern), } - naming.any(|(marker, _)| marker == '-') } -/// The `+`/`-` marker and the path an exact pattern names, when it is one. +/// A pattern's target with a leading `./` removed, unless it carries glob syntax. /// -/// pi compares these as strings rather than globs, after stripping a leading `./` -/// (`normalizeExactPattern`, pi `v0.84.0`, `core/package-manager.ts:656`), which is -/// what makes them decidable from here at all. -fn exact_pattern(pattern: &str) -> Option<(char, &str)> { - let marker = pattern.chars().next()?; - if marker != '+' && marker != '-' { +/// pi strips that prefix before comparing (`normalizeExactPattern`, pi `v0.84.0`, +/// `core/package-manager.ts:656`), which is what makes an exact pattern decidable +/// from here at all. +fn literal_target(target: &str) -> Option<&str> { + if target.contains(['*', '?', '[', '{']) { return None; } - let target = &pattern[marker.len_utf8()..]; - Some((marker, target.strip_prefix("./").unwrap_or(target))) + Some(target.strip_prefix("./").unwrap_or(target)) +} + +/// Whether an exact pattern names this entry. Exact patterns are never globs. +fn target_matches(target: &str, entry: &str) -> bool { + target.strip_prefix("./").unwrap_or(target) == entry } /// The extension entry points a package declares, relative to its own root. /// -/// Empty when they cannot be pinned down: a source that is not on disk, or a +/// Empty when they cannot be pinned down -- a source that is not on disk, or a /// manifest entry carrying a glob or an override marker, which pi expands with -/// `globSync` and `minimatch` rather than comparing as a string. The caller reads -/// empty as "cannot tell" and does not warn. +/// `globSync` rather than comparing as a string. The caller reads empty as +/// "cannot tell" and says so rather than reporting a Pass. fn manifest_extension_entries(path: &Path) -> Vec { let Some(root) = package_root(path) else { return Vec::new(); diff --git a/crates/cli/src/agents/pi/launch.rs b/crates/cli/src/agents/pi/launch.rs index cda9e2af7..668448a4a 100644 --- a/crates/cli/src/agents/pi/launch.rs +++ b/crates/cli/src/agents/pi/launch.rs @@ -91,16 +91,13 @@ pub(crate) fn prepare( // run -- `-a` overrides it, `defaultProjectTrust` can pre-answer it, and "trust this session // only" persists nothing -- so refusing would block every launch in an untrusted project // over a copy that will not load. Say what happens if the project *is* trusted instead. - for site in super::doctor::relay_extension_sites(¤t_dir()) - .iter() - .filter(|site| site.scope == super::doctor::ExtensionScope::Project) - { + for site in super::doctor::project_copies_beside(¤t_dir(), &path) { launch.notes.push(format!( "{} is a second copy of this extension under the project's `.pi/`. pi loads it \ beside the one passed with `-e` whenever the project is trusted, and then every \ turn, tool and inline-shell event is reported twice. Remove it, or run in an \ untrusted project", - site.path.display() + site.display() )); } diff --git a/crates/cli/src/diagnostics/mod.rs b/crates/cli/src/diagnostics/mod.rs index 58dc840de..db99cadf4 100644 --- a/crates/cli/src/diagnostics/mod.rs +++ b/crates/cli/src/diagnostics/mod.rs @@ -526,9 +526,12 @@ fn pi_extension_trust_check(cwd: &Path) -> Check { // Ahead of the project warning, because a project copy must not hide it: the two ungated // copies load whether or not the project is trusted, and that is the louder problem. - if let Some(site) = sites.first() + // Asked of the copy the launcher would choose, not of whichever site sorts first: a copy pi's + // settings switch off is not one of the two that load, and naming it here would report a + // duplicate that does not exist. + if let Some(launched) = crate::agents::pi::doctor::launchable_extension_path(cwd) && let Some(duplicate) = - crate::agents::pi::doctor::conflicting_extension_site(cwd, &site.path) + crate::agents::pi::doctor::conflicting_extension_site(cwd, &launched) { return Check { name: NAME, @@ -538,7 +541,7 @@ fn pi_extension_trust_check(cwd: &Path) -> Check { so both register hooks and every event is reported twice -- each turn is \ closed as superseded by its own duplicate, and the inline-shell gate decides \ one command twice. Keep one copy", - site.path.display(), + launched.display(), duplicate.display() ), }; @@ -562,18 +565,33 @@ fn pi_extension_trust_check(cwd: &Path) -> Check { }; } + use crate::agents::pi::doctor::SettingsFilter; match sites.first() { // Installed, and switched off in pi's own settings. Same silent drop as the trust gate, // from the other direction -- and `-e` ignores those filters, so the launcher still // instruments a session the user's own `pi` runs are missing. - Some(site) if site.disabled_by_settings => Check { + Some(site) if site.filter == SettingsFilter::Excluded => Check { name: NAME, status: Status::Warn, details: format!( "{} is recorded in pi's settings with its extensions filtered off, so pi does not \ - load it -- remove the `extensions` filter, or the `autoload: false`, on \ - that entry. `nemo-relay run --agent pi` is unaffected: it passes `-e`, \ - which applies no settings filter", + load it -- remove the `extensions` filter, or the `autoload: false`, on that \ + entry. `nemo-relay run --agent pi` is unaffected: it passes `-e`, which applies \ + no settings filter", + site.path.display() + ), + }, + // Installed, with a filter this check cannot evaluate: pi matches glob patterns with + // `minimatch`, which is not reimplemented here. Reporting Pass would claim pi loads it, + // and that is the claim this whole check exists to stop being made on no evidence. + Some(site) if site.filter == SettingsFilter::Undecided => Check { + name: NAME, + status: Status::Warn, + details: format!( + "{} is recorded in pi's settings with a glob filter on its extensions, and \ + whether pi loads it cannot be decided from here. Check that filter if NeMo \ + Relay appears to do nothing. `nemo-relay run --agent pi` is unaffected: it \ + passes `-e`, which applies no settings filter", site.path.display() ), }, diff --git a/crates/cli/tests/coverage/agents/launcher_tests.rs b/crates/cli/tests/coverage/agents/launcher_tests.rs index b48886e37..bf8f9bf65 100644 --- a/crates/cli/tests/coverage/agents/launcher_tests.rs +++ b/crates/cli/tests/coverage/agents/launcher_tests.rs @@ -1832,6 +1832,57 @@ fn pi_launch_refuses_when_two_copies_would_load() { ); } +// The note describes a *second* copy, so it must not fire for a project entry pi's settings +// switch off (it never loads) nor for one that canonicalizes to the launched package (pi +// de-duplicates by path, so it loads once). +#[test] +fn a_project_copy_that_cannot_double_the_trace_is_not_noted() { + let _guard = current_dir_lock().lock().unwrap(); + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + let shared = project.join(".pi").join("extensions").join("nemo-relay"); + write_relay_pi_package(&shared); + // A project settings entry pointing at a copy whose extensions are filtered off. + std::fs::create_dir_all(project.join(".pi")).unwrap(); + write_relay_pi_package(&project.join("off")); + std::fs::write( + project.join(".pi").join("settings.json"), + r#"{"packages": [{"source": "../off", "extensions": []}]}"#, + ) + .unwrap(); + let empty_agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&empty_agent_dir).unwrap(); + // Launch the very copy the project holds, so the remaining project site is the same package. + let _env = EnvScope::set(&[ + ( + crate::agents::pi::launch::PI_EXTENSION_PATH_ENV, + Some(shared.as_os_str()), + ), + ( + crate::agents::pi::doctor::PI_AGENT_DIR_ENV, + Some(empty_agent_dir.as_os_str()), + ), + ]); + let previous = std::env::current_dir().unwrap(); + std::env::set_current_dir(&project).unwrap(); + + let prepared = PreparedAgentLaunch::new( + CodingAgent::Pi, + vec!["pi".into()], + "http://127.0.0.1:4040", + &ResolvedConfig::default(), + false, + ); + + std::env::set_current_dir(previous).unwrap(); + let prepared = prepared.expect("the same package reached twice is one copy"); + let notes = prepared.notes.join(" "); + assert!( + !notes.contains("second copy of this extension under the project"), + "neither project site can double the trace: {notes}" + ); +} + // `-e` is never trust-gated. Promoting a project-scoped install to it would run // repository code pi itself declined to trust, which is the failure the preflight // exists to warn about rather than to work around. diff --git a/crates/cli/tests/coverage/agents/pi_doctor_tests.rs b/crates/cli/tests/coverage/agents/pi_doctor_tests.rs index 5e41b0336..54b645c12 100644 --- a/crates/cli/tests/coverage/agents/pi_doctor_tests.rs +++ b/crates/cli/tests/coverage/agents/pi_doctor_tests.rs @@ -357,7 +357,7 @@ fn an_object_form_package_entry_is_found() { assert_eq!(sites.len(), 1, "{sites:?}"); assert_eq!(sites[0].scope, ExtensionScope::User); - assert!(!sites[0].disabled_by_settings); + assert_eq!(sites[0].filter, SettingsFilter::Loads); // Compared canonically: a recorded source is relative to the settings file, so the // resolved path keeps the `..` pi itself would resolve away. assert_eq!( @@ -369,17 +369,14 @@ fn an_object_form_package_entry_is_found() { // A non-empty pattern list is matched against the package manifest, which this // module does not read. Reporting it as loaded is the deliberate direction: a false // negative here is the exact failure the whole module exists to prevent. -// The other side of the same table. A `+` include, a bare name, and anything this module -// cannot decide -- a glob, which pi expands with `minimatch` -- all count as enabled, because -// a false warning costs more than a missing one. +// The other side of the same table: a `+` force-include, a bare include naming the entry, and +// an `autoload: false` delta that adds it back all leave pi loading it. #[test] fn an_object_form_entry_whose_patterns_leave_it_loaded_is_not_reported_as_disabled() { for body in [ r#"{"packages": [{"source": "../checkout", "extensions": ["+index.ts"]}]}"#, r#"{"packages": [{"source": "../checkout", "extensions": ["index.ts"]}]}"#, r#"{"packages": [{"source": "../checkout", "autoload": false, "extensions": ["+index.ts"]}]}"#, - // Deliberate fail-open: pi's glob matcher is not reimplemented here. - r#"{"packages": [{"source": "../checkout", "extensions": ["!*.ts"]}]}"#, ] { let temp = tempfile::tempdir().unwrap(); let agent_dir = temp.path().join("agent"); @@ -391,10 +388,78 @@ fn an_object_form_entry_whose_patterns_leave_it_loaded_is_not_reported_as_disabl let sites = relay_extension_sites(temp.path()); assert_eq!(sites.len(), 1, "{body}: {sites:?}"); - assert!(!sites[0].disabled_by_settings, "{body}"); + assert_eq!(sites[0].filter, SettingsFilter::Loads, "{body}"); + } +} + +// An include list that never names this entry leaves pi loading nothing from the package -- +// step 1 keeps only what the includes match. Decidable without a glob matcher, and previously +// reported as a plain Pass. +#[test] +fn an_include_list_that_omits_this_entry_is_reported_as_disabled() { + for body in [ + r#"{"packages": [{"source": "../checkout", "extensions": ["other.ts"]}]}"#, + r#"{"packages": [{"source": "../checkout", "autoload": false, "extensions": ["+other.ts"]}]}"#, + ] { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("checkout")); + std::fs::write(agent_dir.join("settings.json"), body).unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + assert_eq!(sites[0].filter, SettingsFilter::Excluded, "{body}"); + } +} + +// A glob is pi's to evaluate, not this module's -- but saying Pass would claim pi loads the +// extension on no evidence, which is the claim this whole module exists to stop making. +#[test] +fn a_glob_filter_is_reported_as_undecided_rather_than_as_loaded() { + for body in [ + r#"{"packages": [{"source": "../checkout", "extensions": ["!*.ts"]}]}"#, + r#"{"packages": [{"source": "../checkout", "extensions": ["src/*.ts"]}]}"#, + ] { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("checkout")); + std::fs::write(agent_dir.join("settings.json"), body).unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + assert_eq!(sites[0].filter, SettingsFilter::Undecided, "{body}"); } } +// Choosing the first site regardless of its filter manufactured the duplicate it then refused: +// `-e` re-enables the disabled copy, and the enabled one becomes a genuine second load. +#[test] +fn the_launch_path_prefers_a_copy_pi_already_loads() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("off")); + let live = temp.path().join("live"); + write_relay_package(&live); + std::fs::write( + agent_dir.join("settings.json"), + r#"{"packages": [{"source": "../off", "extensions": []}, "../live"]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let launched = launchable_extension_path(temp.path()).unwrap(); + + assert_eq!( + std::fs::canonicalize(&launched).unwrap(), + std::fs::canonicalize(&live).unwrap(), + "the enabled copy must win, so the disabled one stays disabled" + ); + assert_eq!(conflicting_extension_site(temp.path(), &launched), None); +} + // B1: two copies inside ONE source. pi resolves every distinct package source, so both load // and post every hook twice -- and stopping at the first match hid exactly that. #[test] @@ -475,7 +540,7 @@ fn an_object_form_entry_with_extension_patterns_is_still_found() { let sites = relay_extension_sites(temp.path()); assert_eq!(sites.len(), 1, "{sites:?}"); - assert!(!sites[0].disabled_by_settings); + assert_eq!(sites[0].filter, SettingsFilter::Loads); } // Installed and switched off is not the same as absent, and must not be reported as @@ -504,7 +569,7 @@ fn an_object_form_entry_whose_extensions_are_disabled_is_reported_as_disabled() let sites = relay_extension_sites(temp.path()); assert_eq!(sites.len(), 1, "{body}: {sites:?}"); - assert!(sites[0].disabled_by_settings, "{body}"); + assert_eq!(sites[0].filter, SettingsFilter::Excluded, "{body}"); // Still launchable, deliberately: `-e` applies no settings filter, so the launcher // instruments a session the user's own `pi` runs are missing. assert_eq!( diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index 92dbfed1c..a241f8dd3 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -195,9 +195,11 @@ that gives up. The default is fail-open, so an unreachable gateway does not brick the agent. `NEMO_RELAY_PI_FAIL=closed` opts in to blocking, and the block reason then says explicitly that it is an infrastructure fault rather than a judgment about the -request — and which fault it was, because a gateway that never answered and one -that answered without a decision (a rejected payload, an unreadable body, a 403 -with no guardrail marker) are debugged in different places. +request — and *which* fault, because the four are debugged in four different +places: nothing answered, nothing answered in time (the gateway may be up and +slow), the gateway answered without a decision (a rejected payload, an unreadable +body, a 403 with no guardrail marker), or the gate itself failed before the +gateway was asked. ## Argument Transforms diff --git a/integrations/pi/README.md b/integrations/pi/README.md index dd5446d56..ae3ea4872 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -228,8 +228,10 @@ which pi records exactly as if the command had run: `NEMO_RELAY_PI_FAIL` governs this path too: a gateway that cannot be reached allows the command by default, and refuses it under `closed` with a reason that says explicitly that it is an infrastructure fault rather than a judgment — and -whether the gateway never answered or answered without a decision, so the reader -is not sent to debug a socket that is working. +which of four it was: no answer, no answer *in time*, an answer that was not a +decision, or the gate failing before the gateway was asked. A timeout is not an +unreachable gateway, and neither is a 413, so the reader is not sent to debug a +socket that is working. **A rewritten command is refused, not run.** pi's `user_bash` result can replace the *result* or the execution backend, but never the command — both call sites diff --git a/integrations/pi/index.ts b/integrations/pi/index.ts index 837043a70..422bc3433 100644 --- a/integrations/pi/index.ts +++ b/integrations/pi/index.ts @@ -678,9 +678,9 @@ export default function nemoRelayExtension(pi: ExtensionAPI): void { const detail = error instanceof Error ? error.message : String(error); const fault = resolveFault( config ?? configFromEnv(safeSessionId(ctx)), - // Not `reached`: this is the handler failing, not the gateway. Whatever the - // gateway said, no decision came out of it here. - { kind: 'fault', reached: false, detail: `the inline-shell gate failed: ${detail}` }, + // `handler`, not a transport failure: the gateway was never asked. Saying it could + // not be reached would send the reader to a socket that is fine. + { kind: 'fault', origin: 'handler', detail: `the inline-shell gate failed: ${detail}` }, USER_BASH_TOOL_NAME, ); if (fault.kind === 'block') return { result: refusalResult(fault.reason) }; diff --git a/integrations/pi/src/gateway-client.ts b/integrations/pi/src/gateway-client.ts index f85bc8fc8..d8bc0917f 100644 --- a/integrations/pi/src/gateway-client.ts +++ b/integrations/pi/src/gateway-client.ts @@ -32,13 +32,24 @@ export type HookOutcome = /** * Neither a verdict nor a usable success. * - * `reached` is false when nothing came back -- the connection failed, or nothing - * answered in time -- and true when the gateway did answer and the answer was not - * a decision, such as a rejected payload or a body that will not parse. Both block - * under `NEMO_RELAY_PI_FAIL=closed`, but they send whoever reads the block to two - * different places, so the reason has to say which one happened. + * `origin` says where it went wrong, because all four send whoever reads the + * block somewhere different and they all block identically under + * `NEMO_RELAY_PI_FAIL=closed`. A single "could not be reached" was wrong for + * three of them. */ - | { kind: 'fault'; detail: string; reached: boolean }; + | { kind: 'fault'; detail: string; origin: FaultOrigin }; + +/** + * Where a fault happened, which is what a reader has to know to act on it. + * + * - `transport` -- nothing answered. The gateway is down, or the URL is wrong. + * - `timeout` -- nothing answered *in time*. The gateway may be up and slow, and + * posts are serialized, so a gate also waits out everything queued ahead of it. + * - `response` -- the gateway answered, and the answer was not a decision: a + * rejected payload, an unreadable body, a 403 with no guardrail marker. + * - `handler` -- the gateway was never asked. This extension threw. + */ +export type FaultOrigin = 'transport' | 'timeout' | 'response' | 'handler'; /** The fault arm of {@link HookOutcome}, named so a caller can build one. */ export type HookFault = Extract; @@ -105,7 +116,7 @@ export async function postHook( if (body === null || typeof body !== 'object' || Array.isArray(body)) { return { kind: 'fault', - reached: true, + origin: 'response', detail: 'gateway returned a success body that is not a JSON object', }; } @@ -120,18 +131,19 @@ export async function postHook( } // A 403 without the guardrail marker is an authorization fault, not a // policy decision; do not present it to the model as one. - return { kind: 'fault', reached: true, detail: `gateway returned 403 without a guardrail reason` }; + return { kind: 'fault', origin: 'response', detail: `gateway returned 403 without a guardrail reason` }; } - return { kind: 'fault', reached: true, detail: `gateway returned HTTP ${response.status}` }; + return { kind: 'fault', origin: 'response', detail: `gateway returned HTTP ${response.status}` }; } catch (error) { - const detail = - error instanceof Error && error.name === 'AbortError' - ? `gateway did not respond within ${config.timeoutMs}ms` - : `gateway request failed: ${error instanceof Error ? error.message : String(error)}`; - // Nothing usable came back: a transport failure, or a timeout that may have arrived - // and never answered. Either way there is no response to have misread. - return { kind: 'fault', reached: false, detail }; + const timedOut = error instanceof Error && error.name === 'AbortError'; + const detail = timedOut + ? `gateway did not respond within ${config.timeoutMs}ms` + : `gateway request failed: ${error instanceof Error ? error.message : String(error)}`; + // A timeout is not an unreachable gateway. It may be up and slow, and because posts are + // serialized a gate also waits out everything queued ahead of it -- so the remedy is the + // timeout value or the gateway's speed, not the socket. + return { kind: 'fault', origin: timedOut ? 'timeout' : 'transport', detail }; } finally { clearTimeout(timer); } @@ -161,14 +173,18 @@ export function resolveFault( toolName: string, ): HookOutcome { if (config.onFault === 'open') return { kind: 'allow' }; - // Two openings, one tail. The tail is the part a model has to act on and it is the - // same either way: nothing judged the request, so the request is not what to change. - // The opening differs because "could not be reached", said of a gateway that replied - // 413, sends the reader to debug connectivity -- the one thing that is working. This - // string reaches the model verbatim, so it is also what the user reads. - const opening = fault.reached - ? `The NeMo Relay policy gateway answered this ${toolName} call without a usable decision` - : `The NeMo Relay policy gateway could not be reached to authorize this ${toolName} call`; + // One opening per origin, one tail. The tail is the part a model has to act on and it is + // the same for all four: nothing judged the request, so the request is not what to change. + // The openings differ because they are debugged in four different places, and "could not + // be reached" said of a gateway that replied 413 sends the reader to the one thing that is + // working. This string reaches the model verbatim, so it is also what the user reads. + const openings: Record = { + transport: `The NeMo Relay policy gateway could not be reached to authorize this ${toolName} call`, + timeout: `The NeMo Relay policy gateway did not answer in time to authorize this ${toolName} call`, + response: `The NeMo Relay policy gateway answered this ${toolName} call without a usable decision`, + handler: `The NeMo Relay policy gate failed before it could authorize this ${toolName} call`, + }; + const opening = openings[fault.origin]; return { kind: 'block', reason: diff --git a/integrations/pi/test/gateway-client.test.mjs b/integrations/pi/test/gateway-client.test.mjs index 5877ef672..490ed2569 100644 --- a/integrations/pi/test/gateway-client.test.mjs +++ b/integrations/pi/test/gateway-client.test.mjs @@ -96,7 +96,7 @@ describe('gateway client wire contract', () => { const outcome = await postHook(baseConfig(url), { hook_event_name: name }); assert.equal(outcome.kind, 'fault', `${name} must not be a plain allow`); assert.match(outcome.detail, /not a JSON object/); - assert.equal(outcome.reached, true, `${name} answered; it was not unreachable`); + assert.equal(outcome.origin, 'response', `${name} answered; it was not unreachable`); } }); @@ -130,14 +130,14 @@ describe('gateway client wire contract', () => { // it as one would tell the model a policy considered and refused its call. const outcome = await postHook(baseConfig(url), { hook_event_name: 'naked-403' }); assert.equal(outcome.kind, 'fault'); - assert.equal(outcome.reached, true, 'a refusal is an answer'); + assert.equal(outcome.origin, 'response', 'a refusal is an answer'); }); it('reports a non-403 error status as a fault, not a block', async () => { const outcome = await postHook(baseConfig(url), { hook_event_name: 'boom' }); assert.equal(outcome.kind, 'fault'); assert.match(outcome.detail, /HTTP 500/); - assert.equal(outcome.reached, true); + assert.equal(outcome.origin, 'response'); }); it('times out rather than hanging pi\'s critical path', async () => { @@ -146,7 +146,7 @@ describe('gateway client wire contract', () => { }); assert.equal(outcome.kind, 'fault'); assert.match(outcome.detail, /did not respond within 50ms/); - assert.equal(outcome.reached, false, 'nothing came back to have misread'); + assert.equal(outcome.origin, 'timeout', 'slow is not the same as absent'); }); it('reports an unreachable gateway as a fault', async () => { @@ -155,7 +155,7 @@ describe('gateway client wire contract', () => { hook_event_name: 'tool_call', }); assert.equal(outcome.kind, 'fault'); - assert.equal(outcome.reached, false); + assert.equal(outcome.origin, 'transport'); }); it('sends the session id in both the header and the payload', async () => { @@ -173,7 +173,7 @@ describe('failure policy', () => { it('fails open by default so a dead sidecar does not brick the agent', () => { const outcome = resolveFault( { url: '', timeoutMs: 1, onFault: 'open', sessionId: 's' }, - { kind: 'fault', reached: false, detail: 'connection refused' }, + { kind: 'fault', origin: 'transport', detail: 'connection refused' }, 'read', ); assert.deepEqual(outcome, { kind: 'allow' }); @@ -182,7 +182,7 @@ describe('failure policy', () => { it('fails closed on request, and says the block is infrastructure not policy', () => { const outcome = resolveFault( { url: '', timeoutMs: 1, onFault: 'closed', sessionId: 's' }, - { kind: 'fault', reached: false, detail: 'connection refused' }, + { kind: 'fault', origin: 'transport', detail: 'connection refused' }, 'read', ); assert.equal(outcome.kind, 'block'); @@ -197,7 +197,7 @@ describe('failure policy', () => { it('says the gateway answered when it answered, rather than that it was unreachable', () => { const outcome = resolveFault( { url: '', timeoutMs: 1, onFault: 'closed', sessionId: 's' }, - { kind: 'fault', reached: true, detail: 'gateway returned HTTP 413' }, + { kind: 'fault', origin: 'response', detail: 'gateway returned HTTP 413' }, 'write', ); assert.equal(outcome.kind, 'block'); @@ -207,6 +207,26 @@ describe('failure policy', () => { assert.match(outcome.reason, /infrastructure fault, not a judgment/); assert.match(outcome.reason, /HTTP 413/); }); + + // A timeout is not an unreachable gateway, and a handler failure is not a transport + // result. All four block identically; the sentence is the only thing telling the reader + // which of four places to look. + it('gives each fault origin its own opening', () => { + const config = { url: '', timeoutMs: 1, onFault: 'closed', sessionId: 's' }; + const opening = (origin) => + resolveFault(config, { kind: 'fault', origin, detail: 'd' }, 'read').reason; + + assert.match(opening('transport'), /could not be reached/); + assert.match(opening('timeout'), /did not answer in time/); + assert.match(opening('response'), /without a usable decision/); + assert.match(opening('handler'), /gate failed before it could authorize/); + + const openings = ['transport', 'timeout', 'response', 'handler'].map(opening); + assert.equal(new Set(openings).size, 4, 'each origin must read differently'); + for (const reason of openings) { + assert.match(reason, /infrastructure fault, not a judgment/); + } + }); }); describe('configFromEnv', () => { From 8969d7f2e37eb5b8b7339e2ac6997b63caa314d8 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 15:17:48 -0700 Subject: [PATCH 37/41] fix(pi): find the extension when settings.json lists it directly `settings.json` carries an `extensions` array alongside `packages` -- "local extension file paths or directories" -- in both scopes, and nothing here read it. A user who registered the extension that way got "pi extension not located" from `doctor` and a hard refusal from `nemo-relay run --agent pi`, for a setup pi loads without complaint. It was also invisible to the duplicate check, so it could double-load undetected. Easy to miss, and worth saying why: `SettingsManager::getExtensionPaths()` exists and has no callers, so the key reads as dead until you find the generic loop over pi's four resource types that consumes it by name (`resolve`, pi `v0.84.0`, `core/package-manager.ts:906-931`). I nearly concluded it was dead config on the strength of that grep. Both entry shapes are handled, because pi treats them differently: a *file* entry is the extension, a *directory* entry is a container it walks (`collectResourceFiles` -> `collectAutoExtensionEntries`). A pattern entry (`+`, `-`, `!`) filters the collected set through the same globbing `packages` filters use, over a file set this module does not enumerate pi's way, so any pattern present yields `Undecided` rather than a guess. Found while answering a scoping-doc question about how the extension is registered and activated, which is the honest reason it surfaced now: nobody had enumerated pi's registration routes end to end since the first round. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- crates/cli/src/agents/pi/doctor.rs | 94 ++++++++++++++++++- .../tests/coverage/agents/pi_doctor_tests.rs | 85 +++++++++++++++++ integrations/pi/README.md | 1 + 3 files changed, 179 insertions(+), 1 deletion(-) diff --git a/crates/cli/src/agents/pi/doctor.rs b/crates/cli/src/agents/pi/doctor.rs index 4d7d14750..1e50bd25b 100644 --- a/crates/cli/src/agents/pi/doctor.rs +++ b/crates/cli/src/agents/pi/doctor.rs @@ -205,9 +205,28 @@ pub(crate) fn relay_extension_sites(cwd: &Path) -> Vec { }), ); }; + let listed = |sites: &mut Vec, settings: &Path, base: &Path, scope| { + sites.extend( + relay_extensions_listed_in_settings(settings, base) + .into_iter() + .map(|install| ExtensionSite { + path: install.path, + scope, + filter: install.filter, + }), + ); + }; if let Some(dir) = user_extensions_dir() { discovered(&mut sites, &dir, ExtensionScope::User); } + if let Some(agent_dir) = pi_agent_dir() { + listed( + &mut sites, + &agent_dir.join(PI_SETTINGS_FILE), + &agent_dir, + ExtensionScope::User, + ); + } if let Some(settings) = user_settings_path() { recorded(&mut sites, &settings, ExtensionScope::User); } @@ -216,14 +235,87 @@ pub(crate) fn relay_extension_sites(cwd: &Path) -> Vec { &cwd.join(PI_CONFIG_DIR).join("extensions"), ExtensionScope::Project, ); + let project_config = cwd.join(PI_CONFIG_DIR); + listed( + &mut sites, + &project_config.join(PI_SETTINGS_FILE), + &project_config, + ExtensionScope::Project, + ); recorded( &mut sites, - &cwd.join(PI_CONFIG_DIR).join(PI_SETTINGS_FILE), + &project_config.join(PI_SETTINGS_FILE), ExtensionScope::Project, ); sites } +/// Copies named by `settings.json`'s own `extensions` array, in either scope. +/// +/// A third registration route, and the one easiest to miss: it is a *sibling* of +/// `packages`, read by a generic loop over pi's four resource types +/// (`resolve`, pi `v0.84.0`, `core/package-manager.ts:906-931`) rather than by +/// anything named after extensions -- `SettingsManager::getExtensionPaths` exists +/// and has no callers, which makes the key look dead until that loop is read. +/// Entries resolve against the settings file's own directory, exactly as +/// `packages` entries do. +/// +/// pi treats a *file* entry as the extension and walks a *directory* entry as a +/// container of them (`collectResourceFiles` -> `collectAutoExtensionEntries`, +/// `core/package-manager.ts:618-625`), so both shapes are checked: the entry +/// itself, then its children. +/// +/// A pattern entry (`+`, `-`, `!`) filters the collected set through the same +/// globbing `packages` filters use, over files this module does not enumerate the +/// way pi does -- so any pattern present makes the verdict `Undecided` rather than +/// a guess in either direction. +fn relay_extensions_listed_in_settings(settings: &Path, base: &Path) -> Vec { + let Some(value) = std::fs::read_to_string(settings) + .ok() + .and_then(|raw| serde_json::from_str::(&raw).ok()) + else { + return Vec::new(); + }; + let Some(entries) = value + .get("extensions") + .and_then(serde_json::Value::as_array) + else { + return Vec::new(); + }; + let entries: Vec<&str> = entries + .iter() + .filter_map(serde_json::Value::as_str) + .collect(); + let filtered = entries + .iter() + .any(|entry| entry.starts_with(['+', '-', '!'])); + let filter = if filtered { + SettingsFilter::Undecided + } else { + SettingsFilter::Loads + }; + let mut found = Vec::new(); + for entry in entries + .iter() + .filter(|entry| !entry.starts_with(['+', '-', '!'])) + { + let resolved = base.join(entry); + if is_relay_extension(&resolved) { + found.push(RecordedInstall { + path: resolved, + filter, + }); + continue; + } + found.extend( + relay_entries_in_directory(&resolved) + .into_iter() + .map(|path| RecordedInstall { path, filter }), + ); + } + found +} + /// The path `nemo-relay run --agent pi` hands to `-e`, when there is one. /// /// Explicit first, then user scope -- the order `relay_extension_sites` already diff --git a/crates/cli/tests/coverage/agents/pi_doctor_tests.rs b/crates/cli/tests/coverage/agents/pi_doctor_tests.rs index 54b645c12..5d24ec19f 100644 --- a/crates/cli/tests/coverage/agents/pi_doctor_tests.rs +++ b/crates/cli/tests/coverage/agents/pi_doctor_tests.rs @@ -460,6 +460,91 @@ fn the_launch_path_prefers_a_copy_pi_already_loads() { assert_eq!(conflicting_extension_site(temp.path(), &launched), None); } +// A third registration route, a sibling of `packages` in the same file, read by a generic loop +// over pi's four resource types rather than by anything named after extensions. Missing it meant +// doctor said "not located" and the launcher refused to start for a user pi loads fine. +#[test] +fn an_extensions_entry_in_settings_is_found_in_both_scopes() { + // User scope: entries resolve against the agent directory. + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("checkout")); + std::fs::write( + agent_dir.join("settings.json"), + r#"{"extensions": ["../checkout/index.ts"]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + assert_eq!(sites.len(), 1, "{sites:?}"); + assert_eq!(sites[0].scope, ExtensionScope::User); + assert_eq!(sites[0].filter, SettingsFilter::Loads); + drop(_env); + + // Project scope: entries resolve against `/.pi`, and pi trust-gates them. + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(project.path().join(".pi")).unwrap(); + write_relay_package(&project.path().join("checkout")); + std::fs::write( + project.path().join(".pi").join("settings.json"), + r#"{"extensions": ["../checkout/index.ts"]}"#, + ) + .unwrap(); + let empty_home = project.path().join("home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let _env = scoped(None, Some(empty_home.as_os_str())); + let sites = relay_extension_sites(project.path()); + assert!( + sites + .iter() + .any(|site| site.scope == ExtensionScope::Project), + "a project `extensions` entry is trust-gated and must be reported: {sites:?}" + ); +} + +// pi treats a directory entry as a *container* of extensions and walks it, so a parent directory +// registers the package inside it. +#[test] +fn an_extensions_entry_naming_a_container_directory_is_walked() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + let container = temp.path().join("integrations"); + write_relay_package(&container.join("pi")); + std::fs::write( + agent_dir.join("settings.json"), + r#"{"extensions": ["../integrations"]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + assert_eq!(sites.len(), 1, "{sites:?}"); +} + +// The array filters its collected set with the same globbing `packages` filters use, over files +// this module does not enumerate the way pi does -- so a pattern makes it undecidable, not a Pass. +#[test] +fn an_extensions_entry_carrying_a_pattern_is_reported_as_undecided() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("checkout")); + std::fs::write( + agent_dir.join("settings.json"), + r#"{"extensions": ["../checkout/index.ts", "-index.ts"]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + assert_eq!(sites.len(), 1, "{sites:?}"); + assert_eq!(sites[0].filter, SettingsFilter::Undecided); +} + // B1: two copies inside ONE source. pi resolves every distinct package source, so both load // and post every hook twice -- and stopping at the first match hid exactly that. #[test] diff --git a/integrations/pi/README.md b/integrations/pi/README.md index ae3ea4872..90ba35899 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -57,6 +57,7 @@ pi install /path/to/NeMo-Relay/integrations/pi |---|---|---| | `~/.pi/agent/extensions/` | Yes | No | | `pi install ` | Yes | No | +| `settings.json` `"extensions": [...]` | Yes | Only the project copy | | `-e ` | Per-invocation; what the launcher uses | No | | `.pi/extensions/` or `pi install --local` | **No** | **Yes** | | `pi install ` | **No** — see below | — | From 47e88beb651f8c581e9272533e8c1d6fc4687c52 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 16:05:44 -0700 Subject: [PATCH 38/41] style(pi): drop the warning emoji from the extension's doc comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five `⚠️` markers across four TypeScript and mjs doc comments. The bold lead already carries the emphasis in every one of them -- "**Never throw from the handler.**", "**This is a structural guarantee, not schema validation.**" -- so the glyph added weight to text that was already the loudest thing in the block. Scoped to comments this branch introduced. The `✓`/`✗` in `diagnostics/render.rs` and the configure wizard stay: those are rendered status glyphs in `doctor` output, not decoration, and they predate this branch. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- integrations/pi/src/argument-transform.ts | 2 +- integrations/pi/src/pi-hook-types.ts | 4 ++-- integrations/pi/src/user-bash.ts | 2 +- integrations/pi/test/harness.mjs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/integrations/pi/src/argument-transform.ts b/integrations/pi/src/argument-transform.ts index dc32667cc..c28a40efb 100644 --- a/integrations/pi/src/argument-transform.ts +++ b/integrations/pi/src/argument-transform.ts @@ -30,7 +30,7 @@ * is refused. An object that satisfied the schema before therefore still has * the required keys, of the required types, afterwards. * - * ⚠️ **This is a structural guarantee, not schema validation.** Value-level + * **This is a structural guarantee, not schema validation.** Value-level * constraints -- `pattern`, `enum`, `minimum`, `format` -- are not checked. That * is a choice, not a limit: see point 2. A transform that rewrites a string to * one the schema would reject still executes. diff --git a/integrations/pi/src/pi-hook-types.ts b/integrations/pi/src/pi-hook-types.ts index 8c3d95e12..4d7f5232b 100644 --- a/integrations/pi/src/pi-hook-types.ts +++ b/integrations/pi/src/pi-hook-types.ts @@ -143,7 +143,7 @@ export type ToolCallEvent = { /** * Returning `{block: true}` short-circuits the remaining `tool_call` handlers. * - * ⚠️ Nothing else does. A truthy result without `block` is *retained* but does not + * Nothing else does. A truthy result without `block` is *retained* but does not * stop iteration, so a handler that runs after this one still sees -- and can * still mutate -- the same `input` object, with no re-validation before it * executes. Loading first protects against being pre-empted; it does not make the @@ -178,7 +178,7 @@ export type UserBashEvent = { * keeps the original command. There is no block-and-reason form, so a refusal * is a synthetic failed result -- see `src/user-bash.ts`. * - * ⚠️ Unlike `tool_call`, `emitUserBash` wraps handlers in try/catch, so an + * Unlike `tool_call`, `emitUserBash` wraps handlers in try/catch, so an * exception here fails **open**: pi logs it and runs the command. And the first * handler to return anything at all wins, so an earlier-loading extension can * preempt this one -- `pi -e` loads first, `pi install` loads last. diff --git a/integrations/pi/src/user-bash.ts b/integrations/pi/src/user-bash.ts index 255e8cce6..33167ce7b 100644 --- a/integrations/pi/src/user-bash.ts +++ b/integrations/pi/src/user-bash.ts @@ -28,7 +28,7 @@ * - `cancelled` and `truncated` are false: nothing was started, so nothing was * interrupted, and the message is whole. * - * ⚠️ **Never throw from the handler.** `emitUserBash` wraps handlers in + * **Never throw from the handler.** `emitUserBash` wraps handlers in * try/catch and moves on, so a thrown error fails *open* and the command runs * unchecked -- the opposite of `tool_call`, which has no try/catch and fails * closed. Every path here returns an explicit decision. diff --git a/integrations/pi/test/harness.mjs b/integrations/pi/test/harness.mjs index 569dfb185..f19a9ff11 100644 --- a/integrations/pi/test/harness.mjs +++ b/integrations/pi/test/harness.mjs @@ -4,7 +4,7 @@ /** * Shared test harness: a stub gateway, and a driver that fires pi's hooks. * - * ⚠️ **The two gated hooks resolve competing handlers by different rules, and + * **The two gated hooks resolve competing handlers by different rules, and * the driver has to model each one.** A driver that picks either rule for both * misrepresents one of them, and preemption is exactly what these tests exist * to pin. From f097e069d385c12f2a25c27935da6a97f98fcf52 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 16:27:41 -0700 Subject: [PATCH 39/41] fix(pi): report a pi minor above the verified band as unverified The descriptor comment said the floor is "the version the integration was verified against rather than a lower bound that is expected to keep holding" -- and then the shared validator used it as a plain floor, so `validate_version_output` accepted every stable version above it. `doctor` called pi 0.85.0 supported for a host that can move a hook shape in a minor release, where the symptom is missing spans rather than an error. The comment admitted the semantics were wrong for pi and the code kept them. `AgentDescriptor` gains `verified_through`. `None` for Claude Code and Codex, whose minors are additive, so the floor really is a floor and nothing changes for them. `Some((0, 84))` for pi. **Reported, not enforced.** An upper bound in `validate_version_output` would become `CliError::Launch` at `process::launcher::validate_agent_version` and refuse to start on pi 0.85.0 -- forcing a downgrade of pi to use Relay at all, over a version that has not been shown to be broken. So the band is a third outcome rather than a second error: below the floor errors, 0.84.x passes clean, above warns in `doctor` and logs a warning at launch. Verified against the binary across all five bands, not just in unit tests: 0.83.0 fails "is unsupported"; 0.84.0 and 0.84.9 are clean; 0.85.0 and 1.0.0 warn with the band named. The test also pins that Claude Code and Codex stay silent on a 99.0.0, so adding this field cannot start warning for them by accident. Docs aligned: "0.84.0 or newer" was the claim the code was making and neither was right. pi.mdx, the support matrix and the extension README now say 0.84.x, and say what happens above it. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- crates/cli/src/agents/claude/mod.rs | 1 + crates/cli/src/agents/codex/mod.rs | 1 + crates/cli/src/agents/mod.rs | 29 ++++++++++++++ crates/cli/src/agents/pi/mod.rs | 5 +++ crates/cli/src/diagnostics/mod.rs | 18 ++++++++- crates/cli/src/process/launcher.rs | 17 ++++++-- .../coverage/agents/coding_agent_tests.rs | 40 +++++++++++++++++++ docs/nemo-relay-cli/pi.mdx | 6 ++- docs/reference/support-matrix.mdx | 2 +- integrations/pi/README.md | 4 +- 10 files changed, 116 insertions(+), 7 deletions(-) diff --git a/crates/cli/src/agents/claude/mod.rs b/crates/cli/src/agents/claude/mod.rs index cb39a3295..b171b3811 100644 --- a/crates/cli/src/agents/claude/mod.rs +++ b/crates/cli/src/agents/claude/mod.rs @@ -19,6 +19,7 @@ pub(super) const DESCRIPTOR: AgentDescriptor = AgentDescriptor { hook_path: "/hooks/claude-code", version_product: "Claude Code", minimum_version: (2, 1, 121), + verified_through: None, hook_events: &[ "SessionStart", "UserPromptSubmit", diff --git a/crates/cli/src/agents/codex/mod.rs b/crates/cli/src/agents/codex/mod.rs index 542515982..bf54f8378 100644 --- a/crates/cli/src/agents/codex/mod.rs +++ b/crates/cli/src/agents/codex/mod.rs @@ -20,6 +20,7 @@ pub(super) const DESCRIPTOR: AgentDescriptor = AgentDescriptor { hook_path: "/hooks/codex", version_product: "codex-cli", minimum_version: (0, 143, 0), + verified_through: None, hook_events: &[ "SessionStart", "UserPromptSubmit", diff --git a/crates/cli/src/agents/mod.rs b/crates/cli/src/agents/mod.rs index 2c8f0a3e0..88b722162 100644 --- a/crates/cli/src/agents/mod.rs +++ b/crates/cli/src/agents/mod.rs @@ -28,6 +28,14 @@ pub(super) struct AgentDescriptor { hook_path: &'static str, version_product: &'static str, minimum_version: (u64, u64, u64), + /// The last `major.minor` this integration was actually verified against, for a + /// host whose minor releases can break the hook contract. + /// + /// `None` means the floor behaves like a floor: newer minors are additive and + /// nothing above it needs reporting. `Some` means the opposite -- above it, the + /// integration is untested rather than known-good, and saying so is the whole + /// point, because the failure it guards against is silent. + verified_through: Option<(u64, u64)>, hook_events: &'static [&'static str], } @@ -128,6 +136,27 @@ impl CodingAgent { Ok(version) } + /// Why a version passes the floor and is still not one we have run against. + /// + /// Deliberately not folded into `validate_version_output`: that returns an error, + /// and an error blocks the launch (`process::launcher::validate_agent_version`). + /// Refusing to start on a version that probably works would be worse than the + /// silence it replaces -- the user would have to downgrade the agent to use + /// Relay at all. This reports; it does not gate. + pub(crate) fn unverified_version(self, version: &Version) -> Option { + let (major, minor) = self.descriptor().verified_through?; + if (version.major, version.minor) <= (major, minor) { + return None; + } + Some(format!( + "{product} {version} is newer than the {product} {major}.{minor}.x this integration \ + was verified against; {product} can change hook shapes in a minor release, and the \ + symptom is missing spans rather than an error. Re-verify, or pin {product} {major}.\ + {minor}.x", + product = self.descriptor().version_product, + )) + } + fn parse_version(self, raw: &str) -> Option { match self { Self::ClaudeCode => claude::parse_version(raw), diff --git a/crates/cli/src/agents/pi/mod.rs b/crates/cli/src/agents/pi/mod.rs index 509a21bd2..d23e4b31f 100644 --- a/crates/cli/src/agents/pi/mod.rs +++ b/crates/cli/src/agents/pi/mod.rs @@ -28,6 +28,11 @@ pub(super) const DESCRIPTOR: AgentDescriptor = AgentDescriptor { // channel, so this floor is the version the integration was verified // against rather than a lower bound that is expected to keep holding. minimum_version: (0, 84, 0), + // Which is why the floor alone was a lie by omission: it accepted 0.85.0 as + // "supported" for a host that can move a hook shape in a minor. Below the floor + // is an error, above this is a warning -- untested, not broken, and blocking a + // launch over it would make the user downgrade pi to use Relay at all. + verified_through: Some((0, 84)), // The hooks the extension actually posts to `/hooks/pi`, which is narrower than the set it // registers with pi. `tool_execution_start` is registered but never forwarded -- it fires // before validation and for calls that never execute, so it is used only to remember a tool diff --git a/crates/cli/src/diagnostics/mod.rs b/crates/cli/src/diagnostics/mod.rs index db99cadf4..41ddd9b69 100644 --- a/crates/cli/src/diagnostics/mod.rs +++ b/crates/cli/src/diagnostics/mod.rs @@ -753,14 +753,30 @@ fn apply_agent_version_status( status: &mut Status, details: &mut Vec, ) { + // Three outcomes, not two. A host whose minor releases can move a hook shape has a + // band above the floor that is untested rather than supported, and reporting it as a + // plain pass is what let a newer pi look verified. + let mut unverified = None; let problem = match version { - Some(version) => agent.validate_version_output(version).err(), + Some(version) => match agent.validate_version_output(version) { + Ok(parsed) => { + unverified = agent.unverified_version(&parsed); + None + } + Err(problem) => Some(problem), + }, None if executable_found => Some(format!( "could not determine version; NeMo Relay requires {}", agent.version_requirement() )), None => None, }; + if let Some(unverified) = unverified { + // Warn even when the agent is not the requested target: an untested host is a + // property of the machine, not of what the user asked about. + *status = combine_status(*status, Status::Warn, true); + details.push(unverified); + } if let Some(problem) = problem { *status = combine_status( *status, diff --git a/crates/cli/src/process/launcher.rs b/crates/cli/src/process/launcher.rs index 4e98ff1c7..bddcbbb37 100644 --- a/crates/cli/src/process/launcher.rs +++ b/crates/cli/src/process/launcher.rs @@ -327,10 +327,21 @@ async fn validate_agent_version(agent: CodingAgent, probe: &[String]) -> Result< ))); } let stdout = String::from_utf8_lossy(&output.stdout); - agent + let version = agent .validate_version_output(&stdout) - .map(|_| ()) - .map_err(CliError::Launch) + .map_err(CliError::Launch)?; + // Logged rather than returned: this is not a reason to refuse the launch, and there is no + // note channel here -- `PreparedAgentLaunch` is already built by the time the probe runs. + if let Some(unverified) = agent.unverified_version(&version) { + log::warn!( + target: "nemo_relay.cli", + event = "agent_version_unverified", + agent = agent.as_arg(), + version = version.to_string().as_str(); + "{unverified}" + ); + } + Ok(()) } // Splits a configured command string into argv words for run mode. This intentionally uses simple diff --git a/crates/cli/tests/coverage/agents/coding_agent_tests.rs b/crates/cli/tests/coverage/agents/coding_agent_tests.rs index d06bdb27a..3d598caa6 100644 --- a/crates/cli/tests/coverage/agents/coding_agent_tests.rs +++ b/crates/cli/tests/coverage/agents/coding_agent_tests.rs @@ -55,6 +55,46 @@ fn centralized_minimum_versions_accept_stable_boundaries() { } } +// The floor alone said "supported" for any stable version above it, which is right for a host +// whose minors are additive and wrong for one that can move a hook shape in a minor. pi is the +// second kind: above 0.84.x it is untested, and the symptom of a broken hook shape is missing +// spans rather than an error, so silence is the worst possible report. +#[test] +fn a_minor_above_the_verified_band_is_reported_as_unverified_without_being_rejected() { + let newer = CodingAgent::Pi.validate_version_output("0.85.0").unwrap(); + let note = CodingAgent::Pi + .unverified_version(&newer) + .expect("a newer pi minor must be reported"); + assert!( + note.contains("0.84"), + "the note should name the verified band: {note}" + ); + + // Still accepted: refusing would make a user downgrade pi to use Relay at all. + assert!(CodingAgent::Pi.validate_version_output("0.85.0").is_ok()); + assert!(CodingAgent::Pi.validate_version_output("1.0.0").is_ok()); + + // Inside the band, nothing to say -- including a patch above the floor. + for inside in ["0.84.0", "0.84.7"] { + let version = CodingAgent::Pi.validate_version_output(inside).unwrap(); + assert_eq!( + CodingAgent::Pi.unverified_version(&version), + None, + "{inside}" + ); + } + + // Claude Code and Codex declare no verified band, because their minors are additive. + // Without this, adding a band to the descriptor would silently start warning for them. + for (agent, newer) in [ + (CodingAgent::ClaudeCode, "99.0.0 (Claude Code)"), + (CodingAgent::Codex, "codex-cli 99.0.0"), + ] { + let version = agent.validate_version_output(newer).unwrap(); + assert_eq!(agent.unverified_version(&version), None, "{agent:?}"); + } +} + #[test] fn centralized_minimum_versions_reject_old_prerelease_and_malformed_output() { let cases = [ diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index a241f8dd3..ede12e4cc 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -53,7 +53,11 @@ Three consequences follow from that shape: ## Requirements -Install pi 0.84.0 or newer, and confirm the NeMo Relay extension is reachable: +Install pi 0.84.x, and confirm the NeMo Relay extension is reachable. Anything +below 0.84.0 is rejected; anything above 0.84.x is accepted but reported as +**unverified**, by both `doctor` and the launcher, because pi can change a hook +shape in a minor release and the symptom is missing spans rather than an error: + ```bash nemo-relay doctor pi diff --git a/docs/reference/support-matrix.mdx b/docs/reference/support-matrix.mdx index a72f85a05..4735ab747 100644 --- a/docs/reference/support-matrix.mdx +++ b/docs/reference/support-matrix.mdx @@ -66,7 +66,7 @@ and older CLI versions during installation, diagnostics, and transparent runs. | --- | --- | --- | --- | | Claude Code | 2.1.121 | Persistent install, transparent run, lifecycle hooks, local gateway routing, and pre-tool security | Claude desktop, web, and application sessions are unsupported unless they expose the same local hook and gateway controls. Optimization requires gateway-routed LLM traffic and available hooks. | | Codex CLI | 0.143.0 | Persistent install, transparent run, 10 supported plugin hooks, local gateway routing, and pre-tool security | Cloud or remote tasks that bypass the local machine have partial or no LLM capture. The plugin hook schema has no `SessionEnd`; Relay finalizes the cumulative session snapshot at `Stop`. Encrypted Codex multi-agent v2 payloads cannot be decrypted or reliably linked. | -| pi | 0.84.0 | Transparent run through a Relay-authored pi extension, 15 forwarded event types covering session, turn, tool and inline-shell activity, tool-call security, inline-shell security, and model-call security when redirection applies | Proof of concept. No persistent install: pi has no plugin marketplace. Model traffic is redirected only when the gateway forwards to the endpoint every model of the selected provider would otherwise call, because pi's provider registration is provider-wide; otherwise there are no LLM spans for that model. Seven of pi's 39 built-in providers speak an API the gateway has no route for. Subagents and nested pi processes appear as unrelated sessions. pi ships breaking changes through minor releases, so hook signatures need re-verification after an upgrade. | +| pi | 0.84.x | Transparent run through a Relay-authored pi extension, 15 forwarded event types covering session, turn, tool and inline-shell activity, tool-call security, inline-shell security, and model-call security when redirection applies | Proof of concept. No persistent install: pi has no plugin marketplace. Model traffic is redirected only when the gateway forwards to the endpoint every model of the selected provider would otherwise call, because pi's provider registration is provider-wide; otherwise there are no LLM spans for that model. Seven of pi's 39 built-in providers speak an API the gateway has no route for. Subagents and nested pi processes appear as unrelated sessions. pi ships breaking changes through minor releases, so hook signatures need re-verification after an upgrade; a minor above 0.84.x is accepted but reported as unverified rather than supported. | For installation, diagnostics, and host-specific behavior, refer to [Coding Agent Installation](/nemo-relay-cli/plugin-installation), [Claude diff --git a/integrations/pi/README.md b/integrations/pi/README.md index 90ba35899..8818e0ad8 100644 --- a/integrations/pi/README.md +++ b/integrations/pi/README.md @@ -18,7 +18,9 @@ all policy and all span construction happen in the gateway. Proof of concept. Verified against pi `v0.84.0`. pi ships breaking changes through *minor* releases and has no major-release channel, so re-verify hook -signatures before relying on them. +signatures before relying on them — `nemo-relay doctor pi` and the launcher both +report a minor above 0.84.x as unverified rather than supported, and accept it +anyway, because untested is not the same as broken. **Model traffic is redirected conditionally.** pi has no base-URL flag and no generic environment override — it resolves `baseUrl` per model from a generated From 35ceac9eb5b0024f33074026e2aef0701945353b Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 16:37:31 -0700 Subject: [PATCH 40/41] docs(pi): align the page with the Claude Code and Codex guides The page explained itself by comparison -- a "How It Differs From Codex and Claude Code" section -- which neither sibling guide does, and which dates the page to the moment pi was the new one. Removed, with its load-bearing content kept where a reader looking for that fact would go rather than where the comparison put it: - The sidecar shape moves into the intro, stated on pi's own terms: hooks cannot be injected from outside the process, so they originate inside an extension, and all policy stays in the gateway. - "No persistent plugin install" moves into Requirements, next to the other install routes. - The queued round-trip cost, including the timeout multiplication, moves into Limitations, where an operator choosing `NEMO_RELAY_PI_TIMEOUT_MS` will meet it. - The pointer to Model Redirection is dropped; that section already says it. Structure now matches both siblings: Requirements, Transparent Run, Standalone Gateway, Captured Events, the pi-specific policy sections, Smoke Test, Verify Export, Troubleshoot LLM Lifecycle, Limitations. Two consequences of that: - `Troubleshoot Missing LLM Spans` becomes `Troubleshoot LLM Lifecycle`, the name both siblings use for the same section. - The three limitations that were free-standing top-level sections in the middle of the page -- gate authority, tool-result policy, interrupted sessions -- become subsections of one `Limitations` block at the end, matching Claude Code's `Hook Limitations` and Codex's `Cold-Start Limitation`. Top-level headings drop from 16 to 13. No prose was rewritten beyond the moves and the two paragraphs named above. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- docs/nemo-relay-cli/pi.mdx | 248 +++++++++++++++++-------------------- 1 file changed, 114 insertions(+), 134 deletions(-) diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index ede12e4cc..adb6a3a36 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -14,6 +14,13 @@ model traffic reaches it when the gateway forwards to the endpoint the selected model would otherwise call, which is what [Model Redirection](#model-redirection) explains. +pi has no native hook-configuration file, and its external event stream is +observation-only, so hook calls cannot be injected from outside the process. They +originate inside a pi *extension*, which Relay ships at `integrations/pi/`. The +extension is a thin HTTP client: it forwards pi's lifecycle to `POST /hooks/pi` +and gates tool calls on the gateway's verdict. All policy and all span +construction stay in the gateway. + pi ships breaking changes through *minor* releases and has no major-release channel. The integration is verified against pi `v0.84.0`. Re-verify hook @@ -21,36 +28,6 @@ signatures after a pi upgrade; a silent shape change appears as missing spans, not as an error. -## How It Differs From Codex and Claude Code - -pi has no native hook-configuration file, and its external event stream is -observation-only, so hook calls cannot be injected from outside the process. -They must originate inside a pi *extension*. Relay ships one at -`integrations/pi/`, and it is a thin HTTP client: it forwards pi's lifecycle to -`POST /hooks/pi` and gates tool calls on the gateway's verdict. All policy and -all span construction stay in the gateway. - -Three consequences follow from that shape: - -- There is no persistent plugin install. pi has no plugin marketplace — no - manifest, no `pi plugin` verb, and no MCP client for a plugin-owned server to - serve — so `nemo-relay install pi` is unsupported and says so. Install the - extension with `pi install ` or place it in an auto-discovered - directory (`~/.pi/agent/extensions/`, `.pi/extensions/`). -- The extension pays a round trip per gated tool call. pi awaits extension - handlers on its critical path, so a blocking hook is synchronous by - construction. Observability-only hooks are queued rather than awaited and - drained at shutdown, so they do not charge that path — while the gateway - answers. Posts go out one at a time in hook order, so a gating hook also waits - for whatever is queued ahead of it: against a gateway that holds requests, the - first gate of a session pays `NEMO_RELAY_PI_TIMEOUT_MS` once per queued post, - not once. -- Model traffic is redirected by the extension, not by configuration. pi - resolves `baseUrl` per model from a generated catalog and has no base-URL flag - or generic environment override, so the extension calls - `registerProvider(provider, { baseUrl })` itself — and only when doing so is - safe. See [Model Redirection](#model-redirection). - ## Requirements Install pi 0.84.x, and confirm the NeMo Relay extension is reachable. Anything @@ -58,7 +35,6 @@ below 0.84.0 is rejected; anything above 0.84.x is accepted but reported as **unverified**, by both `doctor` and the launcher, because pi can change a hook shape in a minor release and the symptom is missing spans rather than an error: - ```bash nemo-relay doctor pi ``` @@ -68,6 +44,12 @@ load from and whether pi will trust it, and whether the gateway is answering at the URL the extension posts to. Run it first whenever Relay does not seem to be doing anything. +There is no persistent plugin install. pi has no plugin marketplace — no +manifest, no `pi plugin` verb, and no MCP client for a plugin-owned server to +serve — so `nemo-relay install pi` is unsupported. Install the extension with +`pi install ` or place it in an auto-discovered directory +(`~/.pi/agent/extensions/`, `.pi/extensions/`). + ### Install At User Scope Two routes, both untrusted-project-proof: @@ -107,7 +89,6 @@ path — so pi does not treat it as a failure and never reports it, and the extension cannot report it either, because it is not running. The only symptom is that Relay appears to do nothing. `nemo-relay doctor pi` warns about it. - ## Transparent Run Use the wrapper for no-install local observability. pi has no `nemo-relay pi` @@ -146,7 +127,6 @@ Inspect what would be launched without starting pi: ```bash nemo-relay run --dry-run --print --agent pi ``` - ## Standalone Gateway Run a long-lived gateway and point pi at it by hand: @@ -171,7 +151,46 @@ Run headless pi with `< /dev/null`. pi drains piped standard input before the session starts and returns early only for a TTY, so under an automated harness it can block indefinitely before any hook fires. +## Captured Events + +The extension posts 15 event types, covering session, turn, tool and +inline-shell activity. Twelve carry a pi hook's own name. The other three — +`model_redirect`, `tool_arguments_transformed` and `user_bash_end` — are +synthesized by the extension. The table below names the source of each: +redirection is evaluated at `session_start` and again on every `model_select`, +and `model_redirect` is posted for each decision that explains something — a +redirect, or a skip an operator can act on. Two transient skips are evaluated but +not posted: no model resolved yet, and a provider already pointed at the gateway. +`tool_arguments_transformed` is posted after a rewrite is applied, and +`user_bash_end` after an inline-shell command is decided. + +pi reports both ends of a turn, so Relay opens the turn scope at pi's own +`turn_start` instead of inferring one, and a mark that arrives between turns is +recorded on the session scope rather than opening an empty turn to hold it. +| pi hook | Relay lifecycle | +| --- | --- | +| `session_start` / `session_shutdown` | Session scope. `session_shutdown` is ignored for `reason: "reload"`, which continues the same session | +| `agent_start` / `agent_end` / `agent_settled` | Marks on the session scope | +| `turn_start` / `turn_end` | Turn scope open and close | +| `session_before_compact` | Mark. The compaction is announced, not yet done, and a later extension can still cancel it | +| `session_compact` | Canonical `compaction` mark | +| `session_start`, then every `model_select` | `model_redirect` mark, for each decision that explains something. Synthesized: the decision is re-evaluated per model, so a switch away from a provider the gateway fronts stops redirecting | +| *(after a rewrite)* | `tool_arguments_transformed` mark. Synthesized, so the trace records that the arguments the tool ran were not the ones proposed | +| `tool_call` | Tool span start, and the gate | +| `tool_execution_end` | Tool span end, for every outcome including blocked | +| `user_bash` | Tool span start named `user_bash`, and the inline shell gate | +| *(after `user_bash` completes)* | `user_bash_end`, a tool span end. Synthesized by the extension, because pi reports no completion for inline shell | + +pi's `tool_execution_start` is deliberately not forwarded: it fires before +argument validation and also for calls that never execute. `tool_result` is not +forwarded either, because it does not fire at all for blocked calls and in the +parallel path it fires before `tool_execution_end`. + +A blocked call still produces a well-formed tool span. The guardrail rejects +before the span opens, but pi fires `tool_execution_end` for blocked calls too, +and the gateway synthesizes the missing pair, tagged +`tool_correlation_status: "agent_fallback"`. ## Tool Gating For model-invoked tools, `tool_call` is the only pre-execution decision point @@ -204,7 +223,6 @@ places: nothing answered, nothing answered in time (the gateway may be up and slow), the gateway answered without a decision (a rejected payload, an unreadable body, a 403 with no guardrail marker), or the gate itself failed before the gateway was asked. - ## Argument Transforms A Relay request intercept can rewrite a tool call's arguments. The gateway never @@ -249,7 +267,6 @@ The shipped `examples/rust-native-plugin` demonstrates the mechanism rather than a policy: its tool request intercept tags arguments with `native_plugin_tag` and `native_tool_request_intercept`, which adds keys and so blocks every pi tool call. Enable it to see the refusal, not as a template to copy. - ## Inline Shell Gating pi's bang prefix runs a command outside the tool registry: `!git status` runs it @@ -288,100 +305,6 @@ command — pi reports no completion for inline shell, so the span closes as soo as the verdict arrives and measures the policy round trip. And the hook fires only in the interactive TUI and in RPC mode; headless `-p` has no input loop to type a bang prefix into. - -## Captured Events - -The extension posts 15 event types, covering session, turn, tool and -inline-shell activity. Twelve carry a pi hook's own name. The other three — -`model_redirect`, `tool_arguments_transformed` and `user_bash_end` — are -synthesized by the extension. The table below names the source of each: -redirection is evaluated at `session_start` and again on every `model_select`, -and `model_redirect` is posted for each decision that explains something — a -redirect, or a skip an operator can act on. Two transient skips are evaluated but -not posted: no model resolved yet, and a provider already pointed at the gateway. -`tool_arguments_transformed` is posted after a rewrite is applied, and -`user_bash_end` after an inline-shell command is decided. - -pi reports both ends of a turn, so Relay opens the turn scope at pi's own -`turn_start` instead of inferring one, and a mark that arrives between turns is -recorded on the session scope rather than opening an empty turn to hold it. - -| pi hook | Relay lifecycle | -| --- | --- | -| `session_start` / `session_shutdown` | Session scope. `session_shutdown` is ignored for `reason: "reload"`, which continues the same session | -| `agent_start` / `agent_end` / `agent_settled` | Marks on the session scope | -| `turn_start` / `turn_end` | Turn scope open and close | -| `session_before_compact` | Mark. The compaction is announced, not yet done, and a later extension can still cancel it | -| `session_compact` | Canonical `compaction` mark | -| `session_start`, then every `model_select` | `model_redirect` mark, for each decision that explains something. Synthesized: the decision is re-evaluated per model, so a switch away from a provider the gateway fronts stops redirecting | -| *(after a rewrite)* | `tool_arguments_transformed` mark. Synthesized, so the trace records that the arguments the tool ran were not the ones proposed | -| `tool_call` | Tool span start, and the gate | -| `tool_execution_end` | Tool span end, for every outcome including blocked | -| `user_bash` | Tool span start named `user_bash`, and the inline shell gate | -| *(after `user_bash` completes)* | `user_bash_end`, a tool span end. Synthesized by the extension, because pi reports no completion for inline shell | - -pi's `tool_execution_start` is deliberately not forwarded: it fires before -argument validation and also for calls that never execute. `tool_result` is not -forwarded either, because it does not fire at all for blocked calls and in the -parallel path it fires before `tool_execution_end`. - -A blocked call still produces a well-formed tool span. The guardrail rejects -before the span opens, but pi fires `tool_execution_end` for blocked calls too, -and the gateway synthesizes the missing pair, tagged -`tool_correlation_status: "agent_fallback"`. - -## What The Gate Is Authoritative Over - -pi runs every `tool_call` handler unless one returns `block`, and all of them -share the same mutable `input` object with no re-validation before execution. -Loading with `-e` puts this gate first, which prevents an *earlier* extension -pre-empting it — but an extension loaded after it can rewrite arguments Relay has -already authorized, and those execute unreviewed. - - -pi exposes no ordering API and no post-chain hook, so this cannot be prevented -from inside an extension. In a mixed extension stack, the tool gate is -authoritative over **the model**, not over **the other extensions**. - - -## Tool-Result Policy Is Not Available - -Relay's only tool middleware that can change what a tool *returned* is the -execution intercept, which wraps the callback and so owns execution. pi runs its -tools in its own process and reports the outcome, and the gateway builds spans -from hook posts rather than executing anything, so neither side ever holds that -callback. - - -This is broader than pi. **A tool execution intercept registered by any plugin -does not run under the CLI gateway at all** — the registry's only consumer is -`tool_call_execute`, which the gateway never calls, because it applies policy -through the hook path instead. Conditional-execution guardrails and request -intercepts do run there, since both have standalone runners the gateway invokes -directly; there is no response-phase equivalent to invoke. - - -## Interrupted Sessions - -pi registers **no SIGINT handler in any mode**. All three modes install handlers -for `SIGTERM`, plus `SIGHUP` off Windows, and raw mode is set only by the -interactive TUI — so under `-p`, `--mode json` and `--mode rpc`, Ctrl+C is a real -SIGINT that terminates the process with teardown never running. pi's -`session_shutdown` never fires, and the extension never drains its queue. - -The loss is bounded. Every hook the extension *awaits* has already reached the -gateway: both gates and both turn boundaries block on their round trip. An -interrupt can therefore drop only observability marks queued since the last -awaited hook, and the gateway keeps everything already delivered — the session -scope is left open rather than the trace being lost. - -| Exit | Teardown runs? | -| --- | --- | -| `/quit`, normal completion | Yes | -| `SIGTERM`, `SIGHUP` | Yes | -| `SIGINT` (Ctrl+C in a headless mode) | **No** | -| `SIGKILL`, uncaught exception | **No** | - ## Model Redirection pi has no base-URL flag and no generic environment override — it resolves a base @@ -446,7 +369,6 @@ Point pi at the gateway **root**, not the root plus `/v1`. The Anthropic SDK appends `/v1/messages` itself, while the OpenAI SDK appends `/chat/completions`; the gateway serves both shapes from the root. - ## Agent-Run Attribution One pi prompt can re-enter the agent run several times — provider retry, @@ -476,7 +398,6 @@ strictly within a session. Subagents are not represented. pi has no nested-agent hook of its own, and a child pi process running this extension posts under its own session id, so it appears as an unrelated session rather than as a subagent. - ## Smoke Test Check hook forwarding directly. A 200 is an allow; a 403 with the guardrail @@ -488,7 +409,6 @@ curl -s -w '\nHTTP:%{http_code}\n' -X POST http://127.0.0.1:4040/hooks/pi \ -H 'content-type: application/json' \ -d '{"hook_event_name":"tool_call","session_id":"smoke-pi","tool_call_id":"c1","tool_name":"read","input":{"path":"README.md"}}' ``` - ## Verify Export Complete a pi turn, then confirm the exporter output. With an ATOF file sink @@ -499,8 +419,7 @@ under it. If turn scopes are missing entirely, the extension is not loading. pi collects extension load errors rather than aborting, so a failure is silent — run pi with `-e` pointing at the entry point directly to isolate it. - -## Troubleshoot Missing LLM Spans +## Troubleshoot LLM Lifecycle Look for the `model_redirect` mark on the session scope: it names the outcome and the reason. The common ones are `upstream-mismatch` (start the gateway with @@ -512,3 +431,64 @@ speaks an API the gateway has no route for — pick another model), and — point both upstreams at that provider, or select a model elsewhere. Tool and turn activity are unaffected by any of these. + +## Limitations + +### What The Gate Is Authoritative Over + +pi runs every `tool_call` handler unless one returns `block`, and all of them +share the same mutable `input` object with no re-validation before execution. +Loading with `-e` puts this gate first, which prevents an *earlier* extension +pre-empting it — but an extension loaded after it can rewrite arguments Relay has +already authorized, and those execute unreviewed. + + +pi exposes no ordering API and no post-chain hook, so this cannot be prevented +from inside an extension. In a mixed extension stack, the tool gate is +authoritative over **the model**, not over **the other extensions**. + +### Tool-Result Policy Is Not Available + +Relay's only tool middleware that can change what a tool *returned* is the +execution intercept, which wraps the callback and so owns execution. pi runs its +tools in its own process and reports the outcome, and the gateway builds spans +from hook posts rather than executing anything, so neither side ever holds that +callback. + + +This is broader than pi. **A tool execution intercept registered by any plugin +does not run under the CLI gateway at all** — the registry's only consumer is +`tool_call_execute`, which the gateway never calls, because it applies policy +through the hook path instead. Conditional-execution guardrails and request +intercepts do run there, since both have standalone runners the gateway invokes +directly; there is no response-phase equivalent to invoke. + +### Interrupted Sessions + +pi registers **no SIGINT handler in any mode**. All three modes install handlers +for `SIGTERM`, plus `SIGHUP` off Windows, and raw mode is set only by the +interactive TUI — so under `-p`, `--mode json` and `--mode rpc`, Ctrl+C is a real +SIGINT that terminates the process with teardown never running. pi's +`session_shutdown` never fires, and the extension never drains its queue. + +The loss is bounded. Every hook the extension *awaits* has already reached the +gateway: both gates and both turn boundaries block on their round trip. An +interrupt can therefore drop only observability marks queued since the last +awaited hook, and the gateway keeps everything already delivered — the session +scope is left open rather than the trace being lost. + +| Exit | Teardown runs? | +| --- | --- | +| `/quit`, normal completion | Yes | +| `SIGTERM`, `SIGHUP` | Yes | +| `SIGINT` (Ctrl+C in a headless mode) | **No** | +| `SIGKILL`, uncaught exception | **No** | +### Gateway Round Trips + +pi awaits extension handlers on its critical path, so a blocking hook is +synchronous by construction. Observability-only hooks are queued rather than +awaited and drained at shutdown, so they do not charge that path — while the +gateway answers. Posts go out one at a time in hook order, so a gating hook also +waits for whatever is queued ahead of it: against a gateway that holds requests, +the first gate of a session pays `NEMO_RELAY_PI_TIMEOUT_MS` once per queued post, +not once. From 1c35825dddbbe397848be9c6b6829e9f0f687189 Mon Sep 17 00:00:00 2001 From: Yuchen Zhang Date: Wed, 19 Aug 2026 16:48:31 -0700 Subject: [PATCH 41/41] docs(pi): name what is unavailable, and restore the heading blank lines Two problems, one of them mine from the restructure. **"No persistent plugin install" was the wrong claim.** `pi install ` creates exactly that -- a user-scoped extension pi loads on every later run, and the one the launcher then finds without any variable set. What does not exist is *Relay-managed* installation: `nemo-relay install pi` is unsupported because pi has no marketplace for Relay to install into. Both the pi guide and the support matrix now say that, and say that installing it yourself does persist. **"No-install local observability" was inherited and false here.** The phrase is copied from the Claude Code and Codex guides, where it is true because Relay injects hooks per run. It cannot be true for pi: a hook can only originate inside the extension, so on a clean machine `nemo-relay run --agent pi` fails until the extension is installed, copied, or pointed at. The section now says so, and says the command names the routes that fix it. **Fourteen headings lost their preceding blank line** when the restructure reassembled the page: sections were joined with a single newline, so each heading landed directly against a paragraph, fence, table or ``. Fern renders it, which is why the tests stayed green and the linkcheck passed -- nothing checks this. Restored, and the sibling guides are the reason to care: the source should read the same way across the three. Co-Authored-By: Claude Opus 5 Signed-off-by: Yuchen Zhang --- docs/nemo-relay-cli/pi.mdx | 35 ++++++++++++++++++++++++------- docs/reference/support-matrix.mdx | 2 +- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx index adb6a3a36..758406c05 100644 --- a/docs/nemo-relay-cli/pi.mdx +++ b/docs/nemo-relay-cli/pi.mdx @@ -44,11 +44,15 @@ load from and whether pi will trust it, and whether the gateway is answering at the URL the extension posts to. Run it first whenever Relay does not seem to be doing anything. -There is no persistent plugin install. pi has no plugin marketplace — no -manifest, no `pi plugin` verb, and no MCP client for a plugin-owned server to -serve — so `nemo-relay install pi` is unsupported. Install the extension with -`pi install ` or place it in an auto-discovered directory -(`~/.pi/agent/extensions/`, `.pi/extensions/`). +**Relay-managed installation is unavailable**: `nemo-relay install pi` is +unsupported, because pi has no plugin marketplace — no manifest, no `pi plugin` +verb, and no MCP client for a plugin-owned server to serve — so there is nothing +for Relay to install into. + +Install the extension yourself instead, once. `pi install ` and a copy +into an auto-discovered directory are both **persistent**: pi loads the +extension on every later run, and `nemo-relay run --agent pi` finds it without +any variable being set. ### Install At User Scope @@ -89,10 +93,14 @@ path — so pi does not treat it as a failure and never reports it, and the extension cannot report it either, because it is not running. The only symptom is that Relay appears to do nothing. `nemo-relay doctor pi` warns about it. + ## Transparent Run -Use the wrapper for no-install local observability. pi has no `nemo-relay pi` -shortcut of its own, unlike Claude Code and Codex; run it through `run`: +Use the wrapper for local observability that leaves your pi configuration +untouched. It is **not install-free**: the extension has to be present already, +because a hook can only originate inside it, so run the install above first. On a +clean machine this command fails, and names the routes that fix it. There is no +`nemo-relay pi` shortcut, so use `run`: ```bash nemo-relay run --agent pi @@ -127,6 +135,7 @@ Inspect what would be launched without starting pi: ```bash nemo-relay run --dry-run --print --agent pi ``` + ## Standalone Gateway Run a long-lived gateway and point pi at it by hand: @@ -151,6 +160,7 @@ Run headless pi with `< /dev/null`. pi drains piped standard input before the session starts and returns early only for a TTY, so under an automated harness it can block indefinitely before any hook fires. + ## Captured Events The extension posts 15 event types, covering session, turn, tool and @@ -191,6 +201,7 @@ A blocked call still produces a well-formed tool span. The guardrail rejects before the span opens, but pi fires `tool_execution_end` for blocked calls too, and the gateway synthesizes the missing pair, tagged `tool_correlation_status: "agent_fallback"`. + ## Tool Gating For model-invoked tools, `tool_call` is the only pre-execution decision point @@ -223,6 +234,7 @@ places: nothing answered, nothing answered in time (the gateway may be up and slow), the gateway answered without a decision (a rejected payload, an unreadable body, a 403 with no guardrail marker), or the gate itself failed before the gateway was asked. + ## Argument Transforms A Relay request intercept can rewrite a tool call's arguments. The gateway never @@ -267,6 +279,7 @@ The shipped `examples/rust-native-plugin` demonstrates the mechanism rather than a policy: its tool request intercept tags arguments with `native_plugin_tag` and `native_tool_request_intercept`, which adds keys and so blocks every pi tool call. Enable it to see the refusal, not as a template to copy. + ## Inline Shell Gating pi's bang prefix runs a command outside the tool registry: `!git status` runs it @@ -305,6 +318,7 @@ command — pi reports no completion for inline shell, so the span closes as soo as the verdict arrives and measures the policy round trip. And the hook fires only in the interactive TUI and in RPC mode; headless `-p` has no input loop to type a bang prefix into. + ## Model Redirection pi has no base-URL flag and no generic environment override — it resolves a base @@ -369,6 +383,7 @@ Point pi at the gateway **root**, not the root plus `/v1`. The Anthropic SDK appends `/v1/messages` itself, while the OpenAI SDK appends `/chat/completions`; the gateway serves both shapes from the root. + ## Agent-Run Attribution One pi prompt can re-enter the agent run several times — provider retry, @@ -398,6 +413,7 @@ strictly within a session. Subagents are not represented. pi has no nested-agent hook of its own, and a child pi process running this extension posts under its own session id, so it appears as an unrelated session rather than as a subagent. + ## Smoke Test Check hook forwarding directly. A 200 is an allow; a 403 with the guardrail @@ -409,6 +425,7 @@ curl -s -w '\nHTTP:%{http_code}\n' -X POST http://127.0.0.1:4040/hooks/pi \ -H 'content-type: application/json' \ -d '{"hook_event_name":"tool_call","session_id":"smoke-pi","tool_call_id":"c1","tool_name":"read","input":{"path":"README.md"}}' ``` + ## Verify Export Complete a pi turn, then confirm the exporter output. With an ATOF file sink @@ -419,6 +436,7 @@ under it. If turn scopes are missing entirely, the extension is not loading. pi collects extension load errors rather than aborting, so a failure is silent — run pi with `-e` pointing at the entry point directly to isolate it. + ## Troubleshoot LLM Lifecycle Look for the `model_redirect` mark on the session scope: it names the outcome and @@ -447,6 +465,7 @@ pi exposes no ordering API and no post-chain hook, so this cannot be prevented from inside an extension. In a mixed extension stack, the tool gate is authoritative over **the model**, not over **the other extensions**. + ### Tool-Result Policy Is Not Available Relay's only tool middleware that can change what a tool *returned* is the @@ -463,6 +482,7 @@ through the hook path instead. Conditional-execution guardrails and request intercepts do run there, since both have standalone runners the gateway invokes directly; there is no response-phase equivalent to invoke. + ### Interrupted Sessions pi registers **no SIGINT handler in any mode**. All three modes install handlers @@ -483,6 +503,7 @@ scope is left open rather than the trace being lost. | `SIGTERM`, `SIGHUP` | Yes | | `SIGINT` (Ctrl+C in a headless mode) | **No** | | `SIGKILL`, uncaught exception | **No** | + ### Gateway Round Trips pi awaits extension handlers on its critical path, so a blocking hook is diff --git a/docs/reference/support-matrix.mdx b/docs/reference/support-matrix.mdx index 4735ab747..405854d8a 100644 --- a/docs/reference/support-matrix.mdx +++ b/docs/reference/support-matrix.mdx @@ -66,7 +66,7 @@ and older CLI versions during installation, diagnostics, and transparent runs. | --- | --- | --- | --- | | Claude Code | 2.1.121 | Persistent install, transparent run, lifecycle hooks, local gateway routing, and pre-tool security | Claude desktop, web, and application sessions are unsupported unless they expose the same local hook and gateway controls. Optimization requires gateway-routed LLM traffic and available hooks. | | Codex CLI | 0.143.0 | Persistent install, transparent run, 10 supported plugin hooks, local gateway routing, and pre-tool security | Cloud or remote tasks that bypass the local machine have partial or no LLM capture. The plugin hook schema has no `SessionEnd`; Relay finalizes the cumulative session snapshot at `Stop`. Encrypted Codex multi-agent v2 payloads cannot be decrypted or reliably linked. | -| pi | 0.84.x | Transparent run through a Relay-authored pi extension, 15 forwarded event types covering session, turn, tool and inline-shell activity, tool-call security, inline-shell security, and model-call security when redirection applies | Proof of concept. No persistent install: pi has no plugin marketplace. Model traffic is redirected only when the gateway forwards to the endpoint every model of the selected provider would otherwise call, because pi's provider registration is provider-wide; otherwise there are no LLM spans for that model. Seven of pi's 39 built-in providers speak an API the gateway has no route for. Subagents and nested pi processes appear as unrelated sessions. pi ships breaking changes through minor releases, so hook signatures need re-verification after an upgrade; a minor above 0.84.x is accepted but reported as unverified rather than supported. | +| pi | 0.84.x | Transparent run through a Relay-authored pi extension, 15 forwarded event types covering session, turn, tool and inline-shell activity, tool-call security, inline-shell security, and model-call security when redirection applies | Proof of concept. No Relay-managed install: pi has no plugin marketplace, so `nemo-relay install pi` is unsupported and the extension is installed by the user with `pi install` or a file drop, which does persist. Model traffic is redirected only when the gateway forwards to the endpoint every model of the selected provider would otherwise call, because pi's provider registration is provider-wide; otherwise there are no LLM spans for that model. Seven of pi's 39 built-in providers speak an API the gateway has no route for. Subagents and nested pi processes appear as unrelated sessions. pi ships breaking changes through minor releases, so hook signatures need re-verification after an upgrade; a minor above 0.84.x is accepted but reported as unverified rather than supported. | For installation, diagnostics, and host-specific behavior, refer to [Coding Agent Installation](/nemo-relay-cli/plugin-installation), [Claude