diff --git a/crates/agentplug-host/src/lib.rs b/crates/agentplug-host/src/lib.rs index f79b08e..34f58ff 100644 --- a/crates/agentplug-host/src/lib.rs +++ b/crates/agentplug-host/src/lib.rs @@ -11,7 +11,7 @@ pub use host_state::HostState; pub use imports::{register_env_imports, register_wasi}; pub use install::{install_dir, plugins_dir, wasmtime_cache_dir}; pub use registry::{ - epoch_ticks_for_seconds, read_project_plugin_list, release_shared_plugin, set_gm_pool_size, set_side_plugin_pool_size, DispatchHandle, + epoch_ticks_for_seconds, read_project_plugin_list, refill_shared_plugin, release_shared_plugin, set_gm_pool_size, set_side_plugin_pool_size, DispatchHandle, GmFairnessGuard, ProjectPlugins, EPOCH_TICK_INTERVAL_MS, PLUGIN_IDLE_EVICT_MS, }; diff --git a/crates/agentplug-host/src/registry.rs b/crates/agentplug-host/src/registry.rs index 609c49a..28d765d 100644 --- a/crates/agentplug-host/src/registry.rs +++ b/crates/agentplug-host/src/registry.rs @@ -219,6 +219,20 @@ impl SharedPluginPool { self.slots.iter().any(|s| s.lock().unwrap().is_some()) } + fn all_instantiated(&self) -> bool { + // try_lock: a slot busy mid-dispatch is by definition instantiated. + self.slots.iter().all(|s| match s.try_lock() { + Ok(g) => g.is_some(), + Err(_) => true, + }) + } + + /// The raw slots, for fill-every-empty-slot loads (see `load_plugin` and + /// `refill_shared_plugin`). + pub(crate) fn slots_for_fill(&self) -> &[Arc>>] { + &self.slots + } + fn release_all(&self) -> bool { let mut released = false; for slot in &self.slots { @@ -274,6 +288,32 @@ pub fn release_shared_plugin(plugin_name: &str) -> bool { shared_plugin_pool(plugin_name).release_all() } +/// Re-instantiates every EMPTY slot of a shared plugin's pool from `module`. +/// The plugin-auto-update swap (daemon: `modules.remove` + +/// `release_shared_plugin`) empties the whole pool, and `dispatch` hard-errors +/// on an empty slot with no re-instantiation path of its own — so without an +/// immediate refill every verb fails "plugin X not loaded" until some project +/// happens to re-register (and before the fill-all fix in `load_plugin`, even +/// THAT only refilled one slot). The instantiation `root` is non-binding for +/// shared plugins: `dispatch_on` refreshes the Store's cwd/siblings to the +/// CALLING project before every call. +pub fn refill_shared_plugin(engine: &Engine, plugin_name: &str, module: &Module, root: &Path) -> anyhow::Result { + if !is_stateless_shared_plugin(plugin_name) { + return Ok(0); + } + let pool = shared_plugin_pool(plugin_name); + let mut filled = 0usize; + for slot in pool.slots_for_fill() { + if let Ok(mut guard) = slot.try_lock() { + if guard.is_none() { + *guard = Some(instantiate_plugin(engine, root.to_path_buf(), plugin_name, module)?); + filled += 1; + } + } + } + Ok(filled) +} + /// Runs a verb dispatch against an already-instantiated plugin's OWN Store. /// Shared by `ProjectPlugins::dispatch` (top-level spool dispatch) and /// `host_plugin_call` (cross-plugin dispatch) so both go through the exact @@ -437,7 +477,14 @@ impl ProjectPlugins { } pub fn is_loaded(&self, plugin_name: &str) -> bool { - self.siblings.lock().unwrap().get(plugin_name).map(|p| p.any_instantiated()).unwrap_or(false) + // ALL slots, not ANY: a partially-refilled shared pool (post + // auto-update-swap, pre fill-all `load_plugin`) previously read as + // "loaded" here, so the registration loop skipped the very + // `load_plugin` call that would have healed the remaining empty + // slots — leaving intermittent per-dispatch "plugin X not loaded" + // (whichever concurrent verb drew an empty slot) with no recovery + // short of killing the daemon. + self.siblings.lock().unwrap().get(plugin_name).map(|p| p.all_instantiated()).unwrap_or(false) } /// Instantiates `module` under `plugin_name` for this project. Modules @@ -460,10 +507,22 @@ impl ProjectPlugins { pub fn load_plugin(&mut self, engine: &Engine, plugin_name: &str, module: &Module) -> anyhow::Result<()> { if is_stateless_shared_plugin(plugin_name) { let pool = shared_plugin_pool(plugin_name); - { - let mut guard = pool.acquire().ok_or_else(|| anyhow::anyhow!("plugin {plugin_name} pool busy (timeout acquiring slot for load)"))?; - if guard.is_none() { - *guard = Some(instantiate_plugin(engine, self.root.clone(), plugin_name, module)?); + // Fill EVERY empty slot, not just the first one acquired. After a + // plugin auto-update swap (daemon: modules.remove + + // release_shared_plugin) the whole pool is None; `dispatch` has no + // re-instantiation path (it hard-errors "plugin X not loaded" on + // an empty slot), so a re-registering project that refilled only + // ONE slot left every OTHER slot dead — live-witnessed as + // intermittent per-dispatch "plugin gm not loaded" where exactly + // 1 of N concurrent verbs succeeded (the one that happened to + // acquire the single live slot), unrecoverable short of killing + // the daemon. try_lock per slot: a slot busy mid-dispatch is by + // definition instantiated, so skipping it is correct. + for slot in pool.slots_for_fill() { + if let Ok(mut guard) = slot.try_lock() { + if guard.is_none() { + *guard = Some(instantiate_plugin(engine, self.root.clone(), plugin_name, module)?); + } } } self.siblings.lock().unwrap().insert(plugin_name.to_string(), pool); diff --git a/crates/agentplug-runner/src/daemon.rs b/crates/agentplug-runner/src/daemon.rs index fde5485..346688b 100644 --- a/crates/agentplug-runner/src/daemon.rs +++ b/crates/agentplug-runner/src/daemon.rs @@ -1706,9 +1706,34 @@ fn run_daemon_body(mut plugin_modules: PluginModules) -> anyhow::Result<()> { for (plugin_name, new_version) in pending_plugin_swaps.drain(..) { plugin_modules.modules.remove(&plugin_name); agentplug_host::release_shared_plugin(&plugin_name); - eprintln!( - "[agentplug daemon] refreshed plugin {plugin_name} to {new_version} -- released its Store; next call re-instantiates from the new wasm" - ); + // Recompile from the freshly-downloaded wasm and refill the + // shared pool IMMEDIATELY. Leaving the pool empty here meant + // every subsequent dispatch hard-errored "plugin X not + // loaded" (dispatch has no re-instantiation path), and a + // re-registering project couldn't heal it either: the module + // was gone from `plugin_modules.modules`, so its load loop + // hit the "not yet compiled" skip — live-witnessed as a + // daemon that answered every gm verb with that error for + // hours until it was killed. The refill root is non-binding + // (dispatch_on re-roots the Store per call). + match plugin_modules.get_or_compile(&plugin_name) { + Ok(()) => { + if let Some(module) = plugin_modules.modules.get(&plugin_name) { + let root = std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from(".")); + match agentplug_host::refill_shared_plugin(&plugin_modules.engine, &plugin_name, module, &root) { + Ok(filled) => eprintln!( + "[agentplug daemon] refreshed plugin {plugin_name} to {new_version} -- recompiled and re-instantiated {filled} pool slot(s) from the new wasm" + ), + Err(e) => eprintln!( + "[agentplug daemon] refreshed plugin {plugin_name} to {new_version} but pool refill failed: {e:#} -- next project registration re-instantiates" + ), + } + } + } + Err(e) => eprintln!( + "[agentplug daemon] refreshed plugin {plugin_name} to {new_version} but recompile failed: {e:#} -- next registration retries" + ), + } } }