From 57e85cbcb8f453b310522b0387176c7a50087c4e Mon Sep 17 00:00:00 2001 From: IGN-Styly Date: Sun, 31 May 2026 20:10:51 +0100 Subject: [PATCH 01/28] ic --- Cargo.lock | 5 +++-- engine/src/bin/client.rs | 15 +++++++-------- enginelib/Cargo.toml | 1 + enginelib/src/config.rs | 3 +-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 287b6d6..dd75f28 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -520,9 +520,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -671,6 +671,7 @@ version = "0.2.0" dependencies = [ "chrono", "crossbeam", + "dashmap", "directories", "inventory", "libloading", diff --git a/engine/src/bin/client.rs b/engine/src/bin/client.rs index 546d42e..e8bc800 100644 --- a/engine/src/bin/client.rs +++ b/engine/src/bin/client.rs @@ -1,10 +1,5 @@ use enginelib::{ - Registry, - api::EngineAPI, - events::Events, - event::info, - plugin::LibraryInstance, - prelude::debug, + Registry, api::EngineAPI, event::info, events::Events, plugin::LibraryInstance, prelude::debug, }; use proto::engine_client; use std::{collections::HashMap, error::Error, sync::Arc}; @@ -88,14 +83,18 @@ async fn worker_loop( continue; } Err(status) => { - debug!("worker {}: acquire failed for {}: {:?}", worker_id, task_id, status); + debug!( + "worker {}: acquire failed for {}: {:?}", + worker_id, task_id, status + ); continue; } }; let task_payload = task_req.get_mut(); - let acquired_payload = Arc::new(std::sync::RwLock::new(task_payload.task_payload.clone())); + let acquired_payload = + Arc::new(std::sync::RwLock::new(task_payload.task_payload.clone())); Events::TaskAcquired( api.as_ref(), task_id.clone(), diff --git a/enginelib/Cargo.toml b/enginelib/Cargo.toml index 0481fa3..20341ff 100644 --- a/enginelib/Cargo.toml +++ b/enginelib/Cargo.toml @@ -21,6 +21,7 @@ tokio = { version = "1.50.0", features = ["full"] } postcard = { version = "1.1.3", features = ["use-std"] } inventory = "0.3.22" crossbeam = "0.8.4" +dashmap = "6.2.1" [build-dependencies] vergen-gix = { version = "9.1.0", features = ["build", "cargo", "rustc"] } [profile.release] diff --git a/enginelib/src/config.rs b/enginelib/src/config.rs index 2d46cb5..e3d05e5 100644 --- a/enginelib/src/config.rs +++ b/enginelib/src/config.rs @@ -11,12 +11,11 @@ fn default_clean_tasks() -> u64 { 60 } fn default_task_block_size() -> u32 { - 256 + 256_000 } fn default_pagination_limit() -> u32 { u32::MAX } - #[derive(Debug, Serialize, Deserialize, Clone)] pub struct ConfigTomlServer { #[serde(default)] From 25b3c147b81bea3a91f234382f3c067e3276b2da Mon Sep 17 00:00:00 2001 From: IGN-Styly Date: Sun, 31 May 2026 20:15:08 +0100 Subject: [PATCH 02/28] cleanup --- enginelib/src/api.rs | 116 ------------------------------------------ enginelib/src/task.rs | 5 +- 2 files changed, 3 insertions(+), 118 deletions(-) diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index aab43f8..faee44c 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -27,7 +27,6 @@ pub struct ServerAPI { pub event_bus: EventBus, // RW pub db: sled::Db, // R pub lib_manager: LibraryManager, // RW - pub client: bool, // RW } impl Default for ServerAPI { @@ -49,23 +48,6 @@ impl Default for ServerAPI { } } impl ServerAPI { - // pub fn default_client() -> Self { - // Self { - // cfg: Config::default(), - // task_queue: TaskQueue::default(), - // db: sled::open("engine_client_db").unwrap(), - // lib_manager: LibraryManager::default(), - // task_registry: EngineTaskRegistry::default(), - // event_bus: EventBus { - // event_handler_registry: EngineEventHandlerRegistry { - // event_handlers: HashMap::new(), - // }, - // }, - // solved_tasks: SolvedTasks::default(), - // executing_tasks: ExecutingTaskQueue::default(), - // client: true, - // } - //} pub fn test_default() -> Self { // `sled::Config::temporary(true)` defaults to `/dev/shm` on Linux when no path is set. // Some environments deny writes there, so force a unique temp path. @@ -80,7 +62,6 @@ impl ServerAPI { )); Self { - client: false, cfg: Config::new(), leased_tasks: LeasedTaskQueue::default(), task_queue: TaskQueue::default(), @@ -142,103 +123,6 @@ impl ServerAPI { Some((namespace.to_string(), task.to_string())) } - // pub fn apply_batch_entries( - // db: &sled::Db, - // entries: Vec<(&'static str, Vec)>, - // ) -> sled::Result<()> { - // let mut batch = sled::Batch::default(); - // for (key, value) in entries { - // batch.insert(key, value); - // } - // db.apply_batch(batch) - // } - - // pub fn apply_batch_ops( - // db: &sled::Db, - // ops: Vec<(Vec, Option>)>, - // ) -> sled::Result<()> { - // let mut batch = sled::Batch::default(); - // for (key, value) in ops { - // match value { - // Some(v) => batch.insert(key, v), - // None => batch.remove(key), - // } - // } - // db.apply_batch(batch) - // } - - // pub fn state_op_tasks( - // id: &Identifier, - // value: &Vec, - // ) -> Result<(Vec, Option>), postcard::Error> { - // if value.is_empty() { - // Ok((Self::state_key(Self::TASKS_PREFIX, id), None)) - // } else { - // Ok(( - // Self::state_key(Self::TASKS_PREFIX, id), - // Some(postcard::to_allocvec(value)?), - // )) - // } - // } - - // pub fn state_op_executing( - // id: &Identifier, - // value: &Vec, - // ) -> Result<(Vec, Option>), postcard::Error> { - // if value.is_empty() { - // Ok((Self::state_key(Self::LEASING_PREFIX, id), None)) - // } else { - // Ok(( - // Self::state_key(Self::LEASING_PREFIX, id), - // Some(postcard::to_allocvec(value)?), - // )) - // } - // } - - // pub fn state_op_solved( - // id: &Identifier, - // value: &Vec, - // ) -> Result<(Vec, Option>), postcard::Error> { - // if value.is_empty() { - // Ok((Self::state_key(Self::SOLVED_PREFIX, id), None)) - // } else { - // Ok(( - // Self::state_key(Self::SOLVED_PREFIX, id), - // Some(postcard::to_allocvec(value)?), - // )) - // } - // } - - // pub fn sync_db(api: &mut ServerAPI) { - // // IF THIS FN CAUSES PANIC SOMETHING IS VERY BROKEN - // let mut ops: Vec<(Vec, Option>)> = Vec::new(); - - // for prefix in [ - // Self::TASKS_PREFIX, - // Self::LEASING_PREFIX, - // Self::SOLVED_PREFIX, - // ] { - // for item in api.db.scan_prefix(prefix.as_bytes()) { - // if let Ok((key, _)) = item { - // ops.push((key.to_vec(), None)); - // } - // } - // } - - // for (id, tasks) in &api.task_queue.tasks { - // ops.push(Self::state_op_tasks(id, tasks).unwrap()); - // } - // for (id, tasks) in &api.executing_tasks.tasks { - // ops.push(Self::state_op_executing(id, tasks).unwrap()); - // } - // for (id, tasks) in &api.solved_tasks.tasks { - // ops.push(Self::state_op_solved(id, tasks).unwrap()); - // } - - // Self::apply_batch_ops(&api.db, ops).unwrap(); - // debug!("Synced in-memory state to keyed sled storage"); - // } - fn init_db(api: &mut ServerAPI) { api.task_queue = TaskQueue::default(); api.leased_tasks = LeasedTaskQueue::default(); diff --git a/enginelib/src/task.rs b/enginelib/src/task.rs index 410a90f..35525a3 100644 --- a/enginelib/src/task.rs +++ b/enginelib/src/task.rs @@ -4,6 +4,7 @@ use std::{collections::HashMap, sync::Arc}; use crate::Identifier; use chrono::{DateTime, Utc}; use crossbeam::queue::ArrayQueue; +use dashmap::DashMap; use serde::{Deserialize, Serialize}; use tracing::{error, instrument, warn}; @@ -20,12 +21,12 @@ pub struct LeasedTask { } #[derive(Debug, Default)] pub struct TaskQueue { - pub tasks: HashMap>>, + pub tasks: DashMap>>, } #[derive(Debug, Default, Clone)] pub struct LeasedTaskQueue { - pub tasks: HashMap>, + pub tasks: DashMap>, } pub trait Verifiable { From b111663cc3b94c9107aec0db732c9bd4ad6fa509 Mon Sep 17 00:00:00 2001 From: IGN-Styly Date: Sun, 31 May 2026 20:17:09 +0100 Subject: [PATCH 03/28] cleanup --- enginelib/src/api.rs | 50 +------------------------------------------- 1 file changed, 1 insertion(+), 49 deletions(-) diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index faee44c..d5d6feb 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -43,7 +43,6 @@ impl Default for ServerAPI { }, }, leased_tasks: LeasedTaskQueue::default(), - client: false, } } } @@ -174,7 +173,7 @@ impl ServerAPI { let _ = tracing_subscriber::FmtSubscriber::builder() // all spans/events with a level higher than TRACE (e.g, debug, info, warn, etc.) // will be written to stdout. - .with_max_level(Level::INFO) + .with_max_level(Level::ERROR) // builds the subscriber. .try_init(); }); @@ -236,52 +235,5 @@ pub async fn clear_sled_periodically(api: Arc>, n_minutes: u64 touched_exec.insert(id.clone()); } } - - let mut touched_tasks: HashSet = HashSet::new(); - for (id, task) in moved_tasks { - rw_api - .task_queue - .tasks - .entry(id.clone()) - .or_default() - .push(task); - touched_tasks.insert(id); - } - - if touched_exec.is_empty() && touched_tasks.is_empty() { - continue; - } - - let mut ops: Vec<(Vec, Option>)> = Vec::new(); - - for id in touched_exec { - let value = rw_api - .executing_tasks - .tasks - .get(&id) - .cloned() - .unwrap_or_default(); - match ServerAPI::state_op_executing(&id, &value) { - Ok(op) => ops.push(op), - Err(e) => error!("Failed to serialize executing_tasks entry: {:?}", e), - } - } - - for id in touched_tasks { - let value = rw_api - .task_queue - .tasks - .get(&id) - .cloned() - .unwrap_or_default(); - match ServerAPI::state_op_tasks(&id, &value) { - Ok(op) => ops.push(op), - Err(e) => error!("Failed to serialize tasks entry: {:?}", e), - } - } - - if let Err(e) = ServerAPI::apply_batch_ops(&rw_api.db, ops) { - error!("Failed to update task state in Sled batch: {:?}", e); - } } } From 28c88e6c4681560c097e2dda389915033e932b34 Mon Sep 17 00:00:00 2001 From: IGN-Styly Date: Sun, 31 May 2026 20:44:02 +0100 Subject: [PATCH 04/28] new schema --- Cargo.lock | 49 +++++++++++++++++++++ engine/proto/engine.proto | 21 +++++---- engine/src/bin/server.rs | 2 +- enginelib/Cargo.toml | 1 + enginelib/src/api.rs | 91 ++++++++++++++++----------------------- 5 files changed, 100 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd75f28..1f3eb0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,6 +91,18 @@ dependencies = [ "rustversion", ] +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -378,6 +390,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -669,6 +690,7 @@ dependencies = [ name = "enginelib" version = "0.2.0" dependencies = [ + "async-channel", "chrono", "crossbeam", "dashmap", @@ -704,6 +726,27 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "faster-hex" version = "0.10.0" @@ -2143,6 +2186,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.11.2" diff --git a/engine/proto/engine.proto b/engine/proto/engine.proto index 14a429d..2a581ce 100644 --- a/engine/proto/engine.proto +++ b/engine/proto/engine.proto @@ -1,12 +1,12 @@ syntax = "proto3"; package engine; service Engine { - rpc AquireTask(TaskRequest) returns (Task); + rpc AquireTaskBlock(TaskBlockRequest) returns (TaskBlock); rpc AquireTaskReg(empty) returns (TaskRegistry); - rpc PublishTask(Task) returns (empty); + rpc PublishTaskBlock(TaskBlock) returns (empty); rpc cgrpc(cgrpcmsg) returns (cgrpcmsg); - rpc CreateTask(Task) returns (Task); - rpc DeleteTask(TaskSelector) returns (empty); + rpc CreateTaskBlock(TaskBlock) returns (Task); + rpc DeleteTaskBlock(TaskSelector) returns (empty); rpc GetTasks(TaskPageRequest) returns (TaskPage); rpc CheckAuth(empty) returns (empty); rpc GetMetadata(empty) returns (ServerMetadata); @@ -27,11 +27,13 @@ message TaskSelector { string task = 3; string id = 4; } +message TaskBlockSelector { + repeated TaskBlockSelector selector = 1; +} message empty {} enum TaskState { QUEUED = 0; - PROCESSING = 1; - SOLVED = 2; + SOLVED = 1; } message TaskPageRequest { string namespace = 1; @@ -59,9 +61,9 @@ message TaskRegistry { repeated string tasks = 1; // namespace:task } -message TaskRequest { +message TaskBlockRequest { string task_id = 1; // namespace:task - // bytes payload = 2; + uint32 block_size = 2; } message Task { string id = 4; // the task unique identifier @@ -69,3 +71,6 @@ message Task { string task_id = 2; // namespace:task bytes payload = 3; } +message TaskBlock { + repeated Task tasks = 1; +} diff --git a/engine/src/bin/server.rs b/engine/src/bin/server.rs index 7b70ff8..a7e20c9 100644 --- a/engine/src/bin/server.rs +++ b/engine/src/bin/server.rs @@ -70,7 +70,7 @@ impl Engine for EngineService { request: tonic::Request, ) -> Result, Status> { let challenge = get_auth(&request); - let mut api = self.EngineAPI.write().await; + let mut api = self.EngineAPI.read().await; let db = api.db.clone(); let output = Events::CheckAdminAuth(&mut api, challenge, ("".into(), "".into()), db); if !output { diff --git a/enginelib/Cargo.toml b/enginelib/Cargo.toml index 20341ff..241f162 100644 --- a/enginelib/Cargo.toml +++ b/enginelib/Cargo.toml @@ -22,6 +22,7 @@ postcard = { version = "1.1.3", features = ["use-std"] } inventory = "0.3.22" crossbeam = "0.8.4" dashmap = "6.2.1" +async-channel = "2.5.0" [build-dependencies] vergen-gix = { version = "9.1.0", features = ["build", "cargo", "rustc"] } [profile.release] diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index d5d6feb..4888bc5 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -1,5 +1,6 @@ use chrono::Utc; use crossbeam::queue::ArrayQueue; +use dashmap::DashMap; use tokio::{spawn, sync::RwLock, time::interval}; use tracing::{Level, debug, error, info, instrument}; @@ -135,26 +136,6 @@ impl ServerAPI { } } } - - for item in api.db.scan_prefix(Self::LEASING_PREFIX.as_bytes()) { - if let Ok((key, value)) = item { - if let Some(id) = Self::parse_state_key(Self::LEASING_PREFIX, &key) { - if let Ok(tasks) = postcard::from_bytes::>(&value) { - api.executing_tasks.tasks.insert(id, tasks); - } - } - } - } - - for item in api.db.scan_prefix(Self::SOLVED_PREFIX.as_bytes()) { - if let Ok((key, value)) = item { - if let Some(id) = Self::parse_state_key(Self::SOLVED_PREFIX, &key) { - if let Ok(tasks) = postcard::from_bytes::>(&value) { - api.solved_tasks.tasks.insert(id, tasks); - } - } - } - } } pub fn setup_logger() { @@ -181,7 +162,7 @@ impl ServerAPI { } #[derive(Default, Clone, Debug)] pub struct EngineTaskRegistry { - pub tasks: HashMap>, + pub tasks: DashMap>, } impl Registry for EngineTaskRegistry { #[instrument] @@ -202,38 +183,38 @@ impl Registry for EngineTaskRegistry { pub async fn clear_sled_periodically(api: Arc>, n_minutes: u64) { info!("Sled Cron Job Started"); let mut interval = interval(Duration::from_secs(n_minutes * 60)); - loop { - interval.tick().await; - info!("Purging Unsolved Tasks"); - - let now = Utc::now().timestamp(); - let mut rw_api = api.write().await; - - let mut moved_tasks: Vec<(Identifier, StoredTask)> = Vec::new(); - let mut touched_exec: HashSet = HashSet::new(); - - for (id, task_list) in rw_api.executing_tasks.tasks.iter_mut() { - let before_len = task_list.len(); - task_list.retain(|info| { - let age = now - info.given_at.timestamp(); - if age > 3600 { - info!("Task {:?} is older than an hour! Moving...", info); - moved_tasks.push(( - id.clone(), - StoredTask { - id: info.id.clone(), - bytes: info.bytes.clone(), - }, - )); - false - } else { - true - } - }); - - if task_list.len() != before_len { - touched_exec.insert(id.clone()); - } - } - } + // loop { + // interval.tick().await; + // info!("Purging Unsolved Tasks"); + + // let now = Utc::now().timestamp(); + // let mut rw_api = api.write().await; + + // let mut moved_tasks: Vec<(Identifier, StoredTask)> = Vec::new(); + // let mut touched_exec: HashSet = HashSet::new(); + + // for (id, task_list) in rw_api.executing_tasks.tasks.iter_mut() { + // let before_len = task_list.len(); + // task_list.retain(|info| { + // let age = now - info.given_at.timestamp(); + // if age > 3600 { + // info!("Task {:?} is older than an hour! Moving...", info); + // moved_tasks.push(( + // id.clone(), + // StoredTask { + // id: info.id.clone(), + // bytes: info.bytes.clone(), + // }, + // )); + // false + // } else { + // true + // } + // }); + + // if task_list.len() != before_len { + // touched_exec.insert(id.clone()); + // } + // } + // } } From 5cdbca94c1be932ded4837f20789ff45e4acf069 Mon Sep 17 00:00:00 2001 From: IGN-Styly Date: Sun, 31 May 2026 20:45:19 +0100 Subject: [PATCH 05/28] fn names --- engine/src/bin/server.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/src/bin/server.rs b/engine/src/bin/server.rs index a7e20c9..f486ddb 100644 --- a/engine/src/bin/server.rs +++ b/engine/src/bin/server.rs @@ -79,7 +79,7 @@ impl Engine for EngineService { }; return Ok(tonic::Response::new(proto::Empty {})); } - async fn delete_task( + async fn delete_task_block( &self, request: tonic::Request, ) -> Result, Status> { @@ -414,7 +414,7 @@ impl Engine for EngineService { Ok(tonic::Response::new(response)) } - async fn aquire_task( + async fn aquire_task_block( &self, request: tonic::Request, ) -> Result, tonic::Status> { @@ -526,7 +526,7 @@ impl Engine for EngineService { payload: Vec::new(), })) } - async fn publish_task( + async fn publish_task_block( &self, request: tonic::Request, ) -> Result, tonic::Status> { @@ -659,7 +659,7 @@ impl Engine for EngineService { info!("Task published successfully: {} by user: {}", task_id, uid); Ok(tonic::Response::new(proto::Empty {})) } - async fn create_task( + async fn create_task_block( &self, request: tonic::Request, ) -> Result, tonic::Status> { From fcef51c52388c8a962021ae7cd9e9690a677a46a Mon Sep 17 00:00:00 2001 From: IGN-Styly Date: Sun, 31 May 2026 21:57:05 +0100 Subject: [PATCH 06/28] new queue --- enginelib/src/api.rs | 10 +++------- enginelib/src/task.rs | 8 +++++++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index 4888bc5..5280cd5 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -87,13 +87,9 @@ impl ServerAPI { let mut new_lib_manager = LibraryManager::default(); new_lib_manager.load_modules(api); api.lib_manager = new_lib_manager; - for (id, _tsk) in api.task_registry.tasks.iter() { - api.task_queue - .tasks - .entry(id.clone()) - .insert_entry(ArrayQueue::new( - api.cfg.config_toml.task_block_size as usize, - )); + for (id, _tsk) in api.task_registry.tasks.clone() { + let (s, r) = async_channel::unbounded(); + api.task_queue.tasks.entry(id.clone()).insert((r, s)); api.leased_tasks.tasks.entry(id.clone()).or_default(); } diff --git a/enginelib/src/task.rs b/enginelib/src/task.rs index 35525a3..3f0400f 100644 --- a/enginelib/src/task.rs +++ b/enginelib/src/task.rs @@ -21,7 +21,13 @@ pub struct LeasedTask { } #[derive(Debug, Default)] pub struct TaskQueue { - pub tasks: DashMap>>, + pub tasks: DashMap< + Identifier, + ( + async_channel::Receiver, + async_channel::Sender, + ), + >, } #[derive(Debug, Default, Clone)] From af13e1743aeb54fe80af887a62f2c4c5ae1b6ee1 Mon Sep 17 00:00:00 2001 From: IGN-Styly Date: Sun, 31 May 2026 22:11:30 +0100 Subject: [PATCH 07/28] clarify new data structs --- enginelib/src/api.rs | 21 +++++++++++++++++++++ enginelib/src/task.rs | 9 +++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index 5280cd5..bcf5513 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -119,6 +119,27 @@ impl ServerAPI { Some((namespace.to_string(), task.to_string())) } + fn fill_queue(api: &mut ServerAPI) { + let max_cached = api.cfg.config_toml.task_block_size.max(1) as usize; + for item in api.db.scan_prefix(Self::TASKS_PREFIX.as_bytes()) { + if let Ok((key, value)) = item { + if let Some(id) = Self::parse_state_key(Self::TASKS_PREFIX, &key) { + if let Ok(tasks) = postcard::from_bytes::(&value) { + let is_leased = api + .leased_tasks + .tasks + .get(&id) + .map(|leased| leased.contains_key(&tasks.id)) + .unwrap_or(false); + if !is_leased { + //add to queue + } + } + } + } + } + } + fn init_db(api: &mut ServerAPI) { api.task_queue = TaskQueue::default(); api.leased_tasks = LeasedTaskQueue::default(); diff --git a/enginelib/src/task.rs b/enginelib/src/task.rs index 3f0400f..93726f9 100644 --- a/enginelib/src/task.rs +++ b/enginelib/src/task.rs @@ -19,13 +19,18 @@ pub struct LeasedTask { pub user_id: String, pub given_at: DateTime, } + +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct StoredTaskBlock { + pub tasks: Vec, +} #[derive(Debug, Default)] pub struct TaskQueue { pub tasks: DashMap< Identifier, ( - async_channel::Receiver, - async_channel::Sender, + async_channel::Receiver, + async_channel::Sender, ), >, } From 09f1f8fcb60ddd615b681fe23b10447b92c49e27 Mon Sep 17 00:00:00 2001 From: IGN-Styly Date: Sun, 31 May 2026 22:19:38 +0100 Subject: [PATCH 08/28] derives --- enginelib/src/api.rs | 2 +- enginelib/src/task.rs | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index bcf5513..8ee56e7 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -119,7 +119,7 @@ impl ServerAPI { Some((namespace.to_string(), task.to_string())) } - fn fill_queue(api: &mut ServerAPI) { + fn fill_queue(api: &mut ServerAPI, task_id: Identifier) { let max_cached = api.cfg.config_toml.task_block_size.max(1) as usize; for item in api.db.scan_prefix(Self::TASKS_PREFIX.as_bytes()) { if let Ok((key, value)) = item { diff --git a/enginelib/src/task.rs b/enginelib/src/task.rs index 93726f9..a56de95 100644 --- a/enginelib/src/task.rs +++ b/enginelib/src/task.rs @@ -4,16 +4,16 @@ use std::{collections::HashMap, sync::Arc}; use crate::Identifier; use chrono::{DateTime, Utc}; use crossbeam::queue::ArrayQueue; -use dashmap::DashMap; +use dashmap::{DashMap, DashSet}; use serde::{Deserialize, Serialize}; use tracing::{error, instrument, warn}; -#[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] pub struct StoredTask { pub bytes: Vec, pub id: String, } -#[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] pub struct LeasedTask { pub stored_task: Arc, pub user_id: String, @@ -38,6 +38,7 @@ pub struct TaskQueue { #[derive(Debug, Default, Clone)] pub struct LeasedTaskQueue { pub tasks: DashMap>, + pub tasks_set: DashMap>, } pub trait Verifiable { From 8cc026c3236c6eb5c5e00d475a54fb326134b892 Mon Sep 17 00:00:00 2001 From: IGN-Styly Date: Sun, 31 May 2026 22:48:19 +0100 Subject: [PATCH 09/28] new cfg --- enginelib/src/api.rs | 58 ++++++++++++++++++++++++++++------------- enginelib/src/config.rs | 8 +++++- enginelib/src/task.rs | 1 - 3 files changed, 47 insertions(+), 20 deletions(-) diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index 8ee56e7..94cb112 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -9,7 +9,7 @@ use crate::{ config::Config, event::{EngineEventHandlerRegistry, EventBus}, plugin::LibraryManager, - task::{LeasedTaskQueue, StoredTask, Task, TaskQueue}, + task::{LeasedTaskQueue, StoredTask, StoredTaskBlock, Task, TaskQueue}, }; pub use postcard; pub use postcard::from_bytes; @@ -103,8 +103,8 @@ impl ServerAPI { let t = api.try_read().unwrap().cfg.config_toml.clean_tasks; spawn(clear_sled_periodically(api, t)); } + // type:namespace:task:id const TASKS_PREFIX: &'static str = "tasks:"; - const LEASING_PREFIX: &'static str = "leasing:"; const SOLVED_PREFIX: &'static str = "solved:"; fn state_key(prefix: &str, task_id: &Identifier, id: String) -> Vec { @@ -119,25 +119,47 @@ impl ServerAPI { Some((namespace.to_string(), task.to_string())) } - fn fill_queue(api: &mut ServerAPI, task_id: Identifier) { - let max_cached = api.cfg.config_toml.task_block_size.max(1) as usize; - for item in api.db.scan_prefix(Self::TASKS_PREFIX.as_bytes()) { - if let Ok((key, value)) = item { - if let Some(id) = Self::parse_state_key(Self::TASKS_PREFIX, &key) { - if let Ok(tasks) = postcard::from_bytes::(&value) { - let is_leased = api - .leased_tasks - .tasks - .get(&id) - .map(|leased| leased.contains_key(&tasks.id)) - .unwrap_or(false); - if !is_leased { - //add to queue - } - } + fn fill_queue(api: &ServerAPI, task_id: Identifier) { + let max_block = api.cfg.config_toml.task_block_size.max(1) as usize; + + let Some(channel) = api.task_queue.tasks.get(&task_id) else { + return; + }; + let sender = &channel.1; + + // Build a leased-id set once per call — O(L) instead of O(N·L) per scan item. + let leased: HashSet = api + .leased_tasks + .tasks + .get(&task_id) + .map(|v| v.iter().map(|l| l.stored_task.id.clone()).collect()) + .unwrap_or_default(); + + // Narrower prefix: only this task's records, not every TASKS_PREFIX row. + let prefix = format!("{}{}\u{1f}{}:", Self::TASKS_PREFIX, task_id.0, task_id.1); + let mut block: Vec = Vec::with_capacity(max_block); + + for item in api.db.scan_prefix(prefix.as_bytes()) { + let Ok((_, value)) = item else { continue }; + let Ok(task) = postcard::from_bytes::(&value) else { + continue; + }; + if leased.contains(&task.id) { + continue; + } + + block.push(task); + if block.len() == max_block { + let full = std::mem::replace(&mut block, Vec::with_capacity(max_block)); + if sender.try_send(StoredTaskBlock { tasks: full }).is_err() { + return; // receiver dropped } } } + + if !block.is_empty() { + let _ = sender.try_send(StoredTaskBlock { tasks: block }); + } } fn init_db(api: &mut ServerAPI) { diff --git a/enginelib/src/config.rs b/enginelib/src/config.rs index e3d05e5..5f09076 100644 --- a/enginelib/src/config.rs +++ b/enginelib/src/config.rs @@ -11,11 +11,14 @@ fn default_clean_tasks() -> u64 { 60 } fn default_task_block_size() -> u32 { - 256_000 + 0x8000 } fn default_pagination_limit() -> u32 { u32::MAX } +fn default_task_queue_size() -> u32 { + 2048 +} #[derive(Debug, Serialize, Deserialize, Clone)] pub struct ConfigTomlServer { #[serde(default)] @@ -28,6 +31,8 @@ pub struct ConfigTomlServer { pub pagination_limit: u32, #[serde(default = "default_task_block_size")] pub task_block_size: u32, + #[serde(default = "default_task_queue_size")] + pub task_queue_size: u32, } impl Default for ConfigTomlServer { fn default() -> Self { @@ -37,6 +42,7 @@ impl Default for ConfigTomlServer { clean_tasks: default_clean_tasks(), pagination_limit: default_pagination_limit(), task_block_size: default_task_block_size(), + task_queue_size: default_task_queue_size(), } } } diff --git a/enginelib/src/task.rs b/enginelib/src/task.rs index a56de95..09347f1 100644 --- a/enginelib/src/task.rs +++ b/enginelib/src/task.rs @@ -38,7 +38,6 @@ pub struct TaskQueue { #[derive(Debug, Default, Clone)] pub struct LeasedTaskQueue { pub tasks: DashMap>, - pub tasks_set: DashMap>, } pub trait Verifiable { From abe8f17faf239d253fccf4cf6ccb1aaf5ffc2661 Mon Sep 17 00:00:00 2001 From: IGN-Styly Date: Sun, 31 May 2026 22:53:07 +0100 Subject: [PATCH 10/28] soft cap send by claude --- enginelib/src/api.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index 94cb112..1022075 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -121,12 +121,20 @@ impl ServerAPI { fn fill_queue(api: &ServerAPI, task_id: Identifier) { let max_block = api.cfg.config_toml.task_block_size.max(1) as usize; + let max_queue = api.cfg.config_toml.task_queue_size as usize; let Some(channel) = api.task_queue.tasks.get(&task_id) else { return; }; let sender = &channel.1; + // Soft size lock: bail out if the channel is already at its configured cap. + // The cap is advisory — a partially-built trailing block may still push us + // one block past the limit, but no full block is enqueued once we hit it. + if sender.len() >= max_queue { + return; + } + // Build a leased-id set once per call — O(L) instead of O(N·L) per scan item. let leased: HashSet = api .leased_tasks @@ -154,6 +162,9 @@ impl ServerAPI { if sender.try_send(StoredTaskBlock { tasks: full }).is_err() { return; // receiver dropped } + if sender.len() >= max_queue { + return; + } } } From 4bb7b57551a1dddf3f2870fca01c2f6fde655400 Mon Sep 17 00:00:00 2001 From: IGN-Styly Date: Sun, 31 May 2026 22:59:51 +0100 Subject: [PATCH 11/28] tests --- enginelib/src/api.rs | 10 ---------- enginelib/tests/event_tests.rs | 8 ++++---- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index 1022075..20bd8ed 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -176,16 +176,6 @@ impl ServerAPI { fn init_db(api: &mut ServerAPI) { api.task_queue = TaskQueue::default(); api.leased_tasks = LeasedTaskQueue::default(); - - for item in api.db.scan_prefix(Self::TASKS_PREFIX.as_bytes()) { - if let Ok((key, value)) = item { - if let Some(id) = Self::parse_state_key(Self::TASKS_PREFIX, &key) { - if let Ok(tasks) = postcard::from_bytes::(&value) { - api.task_queue.tasks.get(&id).unwrap().push(Arc::new(tasks)); - } - } - } - } } pub fn setup_logger() { diff --git a/enginelib/tests/event_tests.rs b/enginelib/tests/event_tests.rs index ed5ec9a..9288a3d 100644 --- a/enginelib/tests/event_tests.rs +++ b/enginelib/tests/event_tests.rs @@ -1,4 +1,4 @@ -use enginelib::{Registry, api::EngineAPI, events::ID, task::Verifiable}; +use enginelib::{Registry, api::ServerAPI, events::ID, task::Verifiable}; use macros::Verifiable; use std::sync::Arc; use tracing_test::traced_test; @@ -35,7 +35,7 @@ fn id() { #[traced_test] #[test] fn test_event_registration_and_handling() { - let mut api = EngineAPI::test_default(); + let mut api = ServerAPI::test_default(); let mut test_event = TestEvent { value: 0, @@ -51,7 +51,7 @@ fn test_event_registration_and_handling() { #[traced_test] #[test] fn test_stateful_event_auto_registration() { - let mut api = EngineAPI::test_default(); + let mut api = ServerAPI::test_default(); enginelib::event::register_inventory_handlers(&mut api); @@ -96,7 +96,7 @@ fn test_task_registration() { return Box::new(self.clone()); } } - let mut api = EngineAPI::test_default(); + let mut api = ServerAPI::test_default(); let task_id = ID("test", "test_task"); // Register the task type From 4ff58db628fe813dd2b9a4167eba0080b0477087 Mon Sep 17 00:00:00 2001 From: IGN-Styly Date: Sun, 31 May 2026 23:06:45 +0100 Subject: [PATCH 12/28] new aquire task block --- engine/src/bin/server.rs | 108 +++++++++++++++++---------------------- enginelib/src/api.rs | 2 +- 2 files changed, 47 insertions(+), 63 deletions(-) diff --git a/engine/src/bin/server.rs b/engine/src/bin/server.rs index f486ddb..433632f 100644 --- a/engine/src/bin/server.rs +++ b/engine/src/bin/server.rs @@ -1,5 +1,7 @@ use engine::{get_auth, get_uid}; +use enginelib::api::ServerAPI; use enginelib::plugin::LibraryMetadata; +use enginelib::task::LeasedTask; use enginelib::{ Identifier, RawIdentifier, Registry, api::EngineAPI, @@ -416,8 +418,8 @@ impl Engine for EngineService { async fn aquire_task_block( &self, - request: tonic::Request, - ) -> Result, tonic::Status> { + request: tonic::Request, + ) -> Result, tonic::Status> { let challenge = get_auth(&request); let task_id = request.get_ref().task_id.clone(); let uid = get_uid(&request); @@ -429,7 +431,6 @@ impl Engine for EngineService { { let mut api = self.EngineAPI.write().await; let db = api.db.clone(); - debug!("Validating authentication for task acquisition"); if !Events::CheckAuth(&mut api, uid.clone(), challenge, db) { info!( "Task acquisition denied - invalid authentication for user: {}", @@ -440,91 +441,74 @@ impl Engine for EngineService { } let (namespace, task_name) = task_id.split_once(':').ok_or_else(|| { - info!("Invalid task ID format: {}", task_id); Status::invalid_argument("Invalid task ID format, expected 'namespace:task'") })?; - - debug!("Looking up task definition for {}:{}", namespace, task_name); let key = ID(namespace, task_name); { let api = self.EngineAPI.read().await; if api.task_registry.get(&key).is_none() { - warn!( - "Task acquisition failed - task does not exist: {}:{}", - namespace, task_name - ); return Err(Status::invalid_argument("Task Does not Exist")); } if Events::ServerBeforeTaskAcquire(&api, uid.clone(), task_id.clone()) { - info!( - "ServerBeforeTaskAcquire cancelled for user: {} task: {}", - uid, task_id - ); return Err(Status::aborted( "Task acquire cancelled by server event handler", )); } } - let (ttask, tasks_key_state, exec_key_state, db) = { - let mut api = self.EngineAPI.write().await; - - let queue = api - .task_queue - .tasks - .get_mut(&key) - .ok_or_else(|| Status::not_found("No queued tasks available"))?; - - if queue.is_empty() { - info!("No queued tasks for {}:{}", namespace, task_name); - return Err(Status::not_found("No queued tasks available")); - } - - let ttask = queue.remove(0); - let task_payload = ttask.bytes.clone(); - - api.executing_tasks - .tasks - .entry(key.clone()) - .or_default() - .push(enginelib::task::StoredExecutingTask { - bytes: task_payload, - user_id: uid.clone(), - given_at: Utc::now(), - id: ttask.id.clone(), - }); - - let tasks_key_state = api.task_queue.tasks.get(&key).cloned().unwrap_or_default(); - let exec_key_state = api - .executing_tasks + // Resolve the receiver, refill from sled if drained, then await a block. + // recv blocks until a producer (fill_queue or create_task_block) pushes one, + // so callers should use a gRPC deadline if they need an upper bound. + let receiver = { + let api = self.EngineAPI.read().await; + api.task_queue .tasks .get(&key) - .cloned() - .unwrap_or_default(); - let db = api.db.clone(); - (ttask, tasks_key_state, exec_key_state, db) + .map(|entry| entry.0.clone()) + .ok_or_else(|| Status::not_found("Unknown task type"))? }; - let tasks_op = EngineAPI::state_op_tasks(&key, &tasks_key_state) - .map_err(|e| Status::internal(format!("Serialization error: {}", e)))?; - let exec_op = EngineAPI::state_op_executing(&key, &exec_key_state) - .map_err(|e| Status::internal(format!("Serialization error: {}", e)))?; + if receiver.is_empty() { + let api = self.EngineAPI.read().await; + ServerAPI::fill_queue(&api, key.clone()); + } - EngineAPI::apply_batch_ops(&db, vec![tasks_op, exec_op]) - .map_err(|e| Status::internal(format!("DB insert error: {}", e)))?; + let block = receiver + .recv() + .await + .map_err(|_| Status::unavailable("Task queue closed"))?; + // Lease every task in the block and fire post-acquire events. { let api = self.EngineAPI.read().await; - Events::ServerTaskAcquired(&api, uid.clone(), task_id.clone(), ttask.id.clone()); + let mut entry = api.leased_tasks.tasks.entry(key.clone()).or_default(); + let now = Utc::now(); + for task in &block.tasks { + entry.push(LeasedTask { + stored_task: Arc::new(task.clone()), + user_id: uid.clone(), + given_at: now, + }); + } + drop(entry); + for task in &block.tasks { + Events::ServerTaskAcquired(&api, uid.clone(), task_id.clone(), task.id.clone()); + } } - Ok(tonic::Response::new(proto::Task { - id: ttask.id, - task_id, - task_payload: ttask.bytes, - payload: Vec::new(), - })) + let tasks = block + .tasks + .into_iter() + .map(|t| proto::Task { + id: t.id, + task_id: task_id.clone(), + task_payload: t.bytes, + payload: Vec::new(), + }) + .collect(); + + Ok(tonic::Response::new(proto::TaskBlock { tasks })) } async fn publish_task_block( &self, diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index 20bd8ed..6b8e726 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -119,7 +119,7 @@ impl ServerAPI { Some((namespace.to_string(), task.to_string())) } - fn fill_queue(api: &ServerAPI, task_id: Identifier) { + pub fn fill_queue(api: &ServerAPI, task_id: Identifier) { let max_block = api.cfg.config_toml.task_block_size.max(1) as usize; let max_queue = api.cfg.config_toml.task_queue_size as usize; From e7bf7456c4eb5f310a3f78c554d8649bf69c749a Mon Sep 17 00:00:00 2001 From: IGN-Styly Date: Sun, 31 May 2026 23:36:05 +0100 Subject: [PATCH 13/28] Group server task events into block hooks - Rename task acquire/create/publish events to block-level variants - Update server auth checks to use immutable API borrows --- engine/src/bin/server.rs | 39 ++++++++------ enginelib/src/events/admin_auth_event.rs | 4 +- enginelib/src/events/auth_event.rs | 4 +- enginelib/src/events/mod.rs | 51 ++++++++++++++----- ...rs => server_task_block_acquired_event.rs} | 16 +++--- .../events/server_task_block_created_event.rs | 33 ++++++++++++ ...s => server_task_block_published_event.rs} | 16 +++--- .../src/events/server_task_created_event.rs | 33 ------------ enginelib/src/prelude.rs | 6 +-- 9 files changed, 117 insertions(+), 85 deletions(-) rename enginelib/src/events/{server_task_acquired_event.rs => server_task_block_acquired_event.rs} (51%) create mode 100644 enginelib/src/events/server_task_block_created_event.rs rename enginelib/src/events/{server_task_published_event.rs => server_task_block_published_event.rs} (51%) delete mode 100644 enginelib/src/events/server_task_created_event.rs diff --git a/engine/src/bin/server.rs b/engine/src/bin/server.rs index 433632f..049c97d 100644 --- a/engine/src/bin/server.rs +++ b/engine/src/bin/server.rs @@ -429,9 +429,9 @@ impl Engine for EngineService { ); { - let mut api = self.EngineAPI.write().await; + let api = self.EngineAPI.read().await; let db = api.db.clone(); - if !Events::CheckAuth(&mut api, uid.clone(), challenge, db) { + if !Events::CheckAuth(&api, uid.clone(), challenge, db) { info!( "Task acquisition denied - invalid authentication for user: {}", uid @@ -479,12 +479,14 @@ impl Engine for EngineService { .await .map_err(|_| Status::unavailable("Task queue closed"))?; - // Lease every task in the block and fire post-acquire events. - { + // Lease the block and fire one per-block acquire event. + let instance_ids: Vec = { let api = self.EngineAPI.read().await; let mut entry = api.leased_tasks.tasks.entry(key.clone()).or_default(); let now = Utc::now(); + let mut ids = Vec::with_capacity(block.tasks.len()); for task in &block.tasks { + ids.push(task.id.clone()); entry.push(LeasedTask { stored_task: Arc::new(task.clone()), user_id: uid.clone(), @@ -492,10 +494,10 @@ impl Engine for EngineService { }); } drop(entry); - for task in &block.tasks { - Events::ServerTaskAcquired(&api, uid.clone(), task_id.clone(), task.id.clone()); - } - } + Events::ServerTaskBlockAcquired(&api, uid.clone(), task_id.clone(), ids.clone()); + ids + }; + let _ = instance_ids; let tasks = block .tasks @@ -524,9 +526,9 @@ impl Engine for EngineService { )); { - let mut api = self.EngineAPI.write().await; + let api = self.EngineAPI.read().await; let db = api.db.clone(); - if !Events::CheckAuth(&mut api, uid.clone(), challenge, db) { + if !Events::CheckAuth(&api, uid.clone(), challenge, db) { info!("Aquire Task denied due to Invalid Auth"); return Err(Status::permission_denied("Invalid authentication")); }; @@ -637,7 +639,12 @@ impl Engine for EngineService { { let api = self.EngineAPI.read().await; - Events::ServerTaskPublished(&api, uid.clone(), task_id.clone(), solved_id); + Events::ServerTaskBlockPublished( + &api, + uid.clone(), + task_id.clone(), + vec![solved_id], + ); } info!("Task published successfully: {} by user: {}", task_id, uid); @@ -647,11 +654,11 @@ impl Engine for EngineService { &self, request: tonic::Request, ) -> Result, tonic::Status> { - let mut api = self.EngineAPI.write().await; + let api = self.EngineAPI.read().await; let challenge = get_auth(&request); let uid = get_uid(&request); let db = api.db.clone(); - if !Events::CheckAuth(&mut api, uid, challenge, db) { + if !Events::CheckAuth(&api, uid, challenge, db) { //TODO: change to AdminSpecific Auth info!("Create Task denied due to Invalid Auth"); return Err(Status::permission_denied("Invalid authentication")); @@ -699,11 +706,11 @@ impl Engine for EngineService { if let Err(e) = EngineAPI::apply_batch_ops(&api.db, vec![task_op]) { return Err(Status::internal(format!("DB insert error: {}", e))); } - Events::ServerTaskCreated( + Events::ServerTaskBlockCreated( &api, task_id.clone(), - tbp_tsk.id.clone(), - Arc::new(std::sync::RwLock::new(tbp_tsk.bytes.clone())), + vec![tbp_tsk.id.clone()], + vec![Arc::new(std::sync::RwLock::new(tbp_tsk.bytes.clone()))], ); return Ok(tonic::Response::new(proto::Task { id: tbp_tsk.id.clone(), diff --git a/enginelib/src/events/admin_auth_event.rs b/enginelib/src/events/admin_auth_event.rs index c13533d..a1ce7d2 100644 --- a/enginelib/src/events/admin_auth_event.rs +++ b/enginelib/src/events/admin_auth_event.rs @@ -19,7 +19,7 @@ pub struct AdminAuthEvent { impl AdminAuthEvent { pub fn fire( - api: &mut ServerAPI, + api: &ServerAPI, payload: String, target: Identifier, db: Db, @@ -35,7 +35,7 @@ impl AdminAuthEvent { }); } - pub fn check(api: &mut ServerAPI, payload: String, target: Identifier, db: Db) -> bool { + pub fn check(api: &ServerAPI, payload: String, target: Identifier, db: Db) -> bool { let output = Arc::new(RwLock::new(false)); Self::fire(api, payload, target, db, output.clone()); *output.read().unwrap() diff --git a/enginelib/src/events/auth_event.rs b/enginelib/src/events/auth_event.rs index 1056a99..cd5fa0e 100644 --- a/enginelib/src/events/auth_event.rs +++ b/enginelib/src/events/auth_event.rs @@ -16,7 +16,7 @@ pub struct AuthEvent { } impl AuthEvent { pub fn fire( - api: &mut ServerAPI, + api: &ServerAPI, uid: String, challenge: String, db: Db, @@ -32,7 +32,7 @@ impl AuthEvent { }); } - pub fn check(api: &mut ServerAPI, uid: String, challenge: String, db: Db) -> bool { + pub fn check(api: &ServerAPI, uid: String, challenge: String, db: Db) -> bool { let output = Arc::new(RwLock::new(false)); Self::fire(api, uid, challenge, db, output.clone()); *output.read().unwrap() diff --git a/enginelib/src/events/mod.rs b/enginelib/src/events/mod.rs index 48ee7a5..7c687d6 100644 --- a/enginelib/src/events/mod.rs +++ b/enginelib/src/events/mod.rs @@ -10,9 +10,9 @@ pub mod server_before_task_acquire_event; pub mod server_before_task_create_event; pub mod server_before_task_publish_event; pub mod server_start_event; -pub mod server_task_acquired_event; -pub mod server_task_created_event; -pub mod server_task_published_event; +pub mod server_task_block_acquired_event; +pub mod server_task_block_created_event; +pub mod server_task_block_published_event; pub mod start_event; pub mod task_acquired_event; use std::collections::HashMap; @@ -32,12 +32,12 @@ pub struct Events; impl Events { pub fn init_auth(_api: &mut ServerAPI) {} - pub fn CheckAuth(api: &mut ServerAPI, uid: String, challenge: String, db: Db) -> bool { + pub fn CheckAuth(api: &ServerAPI, uid: String, challenge: String, db: Db) -> bool { auth_event::AuthEvent::check(api, uid, challenge, db) } pub fn CheckAdminAuth( - api: &mut ServerAPI, + api: &ServerAPI, payload: String, target: Identifier, db: Db, @@ -109,21 +109,36 @@ impl Events { server_before_task_create_event::ServerBeforeTaskCreateEvent::check(api, task_id, payload) } - pub fn ServerTaskCreated( + pub fn ServerTaskBlockCreated( api: &ServerAPI, task_id: String, - instance_id: String, - payload: Arc>>, + instance_ids: Vec, + payloads: Vec>>>, ) { - server_task_created_event::ServerTaskCreatedEvent::fire(api, task_id, instance_id, payload) + server_task_block_created_event::ServerTaskBlockCreatedEvent::fire( + api, + task_id, + instance_ids, + payloads, + ) } pub fn ServerBeforeTaskAcquire(api: &ServerAPI, uid: String, task_id: String) -> bool { server_before_task_acquire_event::ServerBeforeTaskAcquireEvent::check(api, uid, task_id) } - pub fn ServerTaskAcquired(api: &ServerAPI, uid: String, task_id: String, instance_id: String) { - server_task_acquired_event::ServerTaskAcquiredEvent::fire(api, uid, task_id, instance_id) + pub fn ServerTaskBlockAcquired( + api: &ServerAPI, + uid: String, + task_id: String, + instance_ids: Vec, + ) { + server_task_block_acquired_event::ServerTaskBlockAcquiredEvent::fire( + api, + uid, + task_id, + instance_ids, + ) } pub fn ServerBeforeTaskPublish( @@ -142,7 +157,17 @@ impl Events { ) } - pub fn ServerTaskPublished(api: &ServerAPI, uid: String, task_id: String, instance_id: String) { - server_task_published_event::ServerTaskPublishedEvent::fire(api, uid, task_id, instance_id) + pub fn ServerTaskBlockPublished( + api: &ServerAPI, + uid: String, + task_id: String, + instance_ids: Vec, + ) { + server_task_block_published_event::ServerTaskBlockPublishedEvent::fire( + api, + uid, + task_id, + instance_ids, + ) } } diff --git a/enginelib/src/events/server_task_acquired_event.rs b/enginelib/src/events/server_task_block_acquired_event.rs similarity index 51% rename from enginelib/src/events/server_task_acquired_event.rs rename to enginelib/src/events/server_task_block_acquired_event.rs index 116ecf1..4d3f7ce 100644 --- a/enginelib/src/events/server_task_acquired_event.rs +++ b/enginelib/src/events/server_task_block_acquired_event.rs @@ -3,23 +3,23 @@ use macros::Event; use crate::{Identifier, api::ServerAPI}; #[derive(Clone, Debug, Event)] -#[event(namespace = "server", name = "task_acquired")] -pub struct ServerTaskAcquiredEvent { +#[event(namespace = "server", name = "task_block_acquired")] +pub struct ServerTaskBlockAcquiredEvent { pub cancelled: bool, pub id: Identifier, pub uid: String, pub task_id: String, - pub instance_id: String, + pub instance_ids: Vec, } -impl ServerTaskAcquiredEvent { - pub fn fire(api: &ServerAPI, uid: String, task_id: String, instance_id: String) { - let mut event = ServerTaskAcquiredEvent { +impl ServerTaskBlockAcquiredEvent { + pub fn fire(api: &ServerAPI, uid: String, task_id: String, instance_ids: Vec) { + let mut event = ServerTaskBlockAcquiredEvent { cancelled: false, - id: ("server".to_string(), "task_acquired".to_string()), + id: ("server".to_string(), "task_block_acquired".to_string()), uid, task_id, - instance_id, + instance_ids, }; api.event_bus.fire(&mut event); } diff --git a/enginelib/src/events/server_task_block_created_event.rs b/enginelib/src/events/server_task_block_created_event.rs new file mode 100644 index 0000000..9f5a331 --- /dev/null +++ b/enginelib/src/events/server_task_block_created_event.rs @@ -0,0 +1,33 @@ +use std::sync::{Arc, RwLock}; + +use macros::Event; + +use crate::{Identifier, api::ServerAPI}; + +#[derive(Clone, Debug, Event)] +#[event(namespace = "server", name = "task_block_created")] +pub struct ServerTaskBlockCreatedEvent { + pub cancelled: bool, + pub id: Identifier, + pub task_id: String, + pub instance_ids: Vec, + pub payloads: Vec>>>, +} + +impl ServerTaskBlockCreatedEvent { + pub fn fire( + api: &ServerAPI, + task_id: String, + instance_ids: Vec, + payloads: Vec>>>, + ) { + let mut event = ServerTaskBlockCreatedEvent { + cancelled: false, + id: ("server".to_string(), "task_block_created".to_string()), + task_id, + instance_ids, + payloads, + }; + api.event_bus.fire(&mut event); + } +} diff --git a/enginelib/src/events/server_task_published_event.rs b/enginelib/src/events/server_task_block_published_event.rs similarity index 51% rename from enginelib/src/events/server_task_published_event.rs rename to enginelib/src/events/server_task_block_published_event.rs index 5e6ee8a..43b997e 100644 --- a/enginelib/src/events/server_task_published_event.rs +++ b/enginelib/src/events/server_task_block_published_event.rs @@ -3,23 +3,23 @@ use macros::Event; use crate::{Identifier, api::ServerAPI}; #[derive(Clone, Debug, Event)] -#[event(namespace = "server", name = "task_published")] -pub struct ServerTaskPublishedEvent { +#[event(namespace = "server", name = "task_block_published")] +pub struct ServerTaskBlockPublishedEvent { pub cancelled: bool, pub id: Identifier, pub uid: String, pub task_id: String, - pub instance_id: String, + pub instance_ids: Vec, } -impl ServerTaskPublishedEvent { - pub fn fire(api: &ServerAPI, uid: String, task_id: String, instance_id: String) { - let mut event = ServerTaskPublishedEvent { +impl ServerTaskBlockPublishedEvent { + pub fn fire(api: &ServerAPI, uid: String, task_id: String, instance_ids: Vec) { + let mut event = ServerTaskBlockPublishedEvent { cancelled: false, - id: ("server".to_string(), "task_published".to_string()), + id: ("server".to_string(), "task_block_published".to_string()), uid, task_id, - instance_id, + instance_ids, }; api.event_bus.fire(&mut event); } diff --git a/enginelib/src/events/server_task_created_event.rs b/enginelib/src/events/server_task_created_event.rs deleted file mode 100644 index 1326818..0000000 --- a/enginelib/src/events/server_task_created_event.rs +++ /dev/null @@ -1,33 +0,0 @@ -use std::sync::{Arc, RwLock}; - -use macros::Event; - -use crate::{Identifier, api::ServerAPI}; - -#[derive(Clone, Debug, Event)] -#[event(namespace = "server", name = "task_created")] -pub struct ServerTaskCreatedEvent { - pub cancelled: bool, - pub id: Identifier, - pub task_id: String, - pub instance_id: String, - pub payload: Arc>>, -} - -impl ServerTaskCreatedEvent { - pub fn fire( - api: &ServerAPI, - task_id: String, - instance_id: String, - payload: Arc>>, - ) { - let mut event = ServerTaskCreatedEvent { - cancelled: false, - id: ("server".to_string(), "task_created".to_string()), - task_id, - instance_id, - payload, - }; - api.event_bus.fire(&mut event); - } -} diff --git a/enginelib/src/prelude.rs b/enginelib/src/prelude.rs index bee62a5..c1af52d 100644 --- a/enginelib/src/prelude.rs +++ b/enginelib/src/prelude.rs @@ -22,9 +22,9 @@ pub use crate::events::server_before_task_acquire_event::ServerBeforeTaskAcquire pub use crate::events::server_before_task_create_event::ServerBeforeTaskCreateEvent; pub use crate::events::server_before_task_publish_event::ServerBeforeTaskPublishEvent; pub use crate::events::server_start_event::ServerStartEvent; -pub use crate::events::server_task_acquired_event::ServerTaskAcquiredEvent; -pub use crate::events::server_task_created_event::ServerTaskCreatedEvent; -pub use crate::events::server_task_published_event::ServerTaskPublishedEvent; +pub use crate::events::server_task_block_acquired_event::ServerTaskBlockAcquiredEvent; +pub use crate::events::server_task_block_created_event::ServerTaskBlockCreatedEvent; +pub use crate::events::server_task_block_published_event::ServerTaskBlockPublishedEvent; pub use crate::events::start_event::StartEvent; pub use crate::events::task_acquired_event::TaskAcquiredEvent; pub use crate::events::{Events, ID}; From 00d06dfd37521e9af3cfe41e79f28bde84bca561 Mon Sep 17 00:00:00 2001 From: IGN-Styly Date: Mon, 1 Jun 2026 18:39:16 +0100 Subject: [PATCH 14/28] Refactor task block persistence and lease handling - Change CreateTaskBlock/PublishTaskBlock to work on TaskBlock payloads - Add LEASED task state and move queued/solved access to sled helpers - Use direct DB helpers for queued/solved deletes, scans, and inserts --- engine/proto/engine.proto | 3 +- engine/src/bin/server.rs | 599 ++++++++++++++++---------------------- enginelib/src/api.rs | 70 ++++- 3 files changed, 307 insertions(+), 365 deletions(-) diff --git a/engine/proto/engine.proto b/engine/proto/engine.proto index 2a581ce..1a2f5c4 100644 --- a/engine/proto/engine.proto +++ b/engine/proto/engine.proto @@ -5,7 +5,7 @@ service Engine { rpc AquireTaskReg(empty) returns (TaskRegistry); rpc PublishTaskBlock(TaskBlock) returns (empty); rpc cgrpc(cgrpcmsg) returns (cgrpcmsg); - rpc CreateTaskBlock(TaskBlock) returns (Task); + rpc CreateTaskBlock(TaskBlock) returns (TaskBlock); rpc DeleteTaskBlock(TaskSelector) returns (empty); rpc GetTasks(TaskPageRequest) returns (TaskPage); rpc CheckAuth(empty) returns (empty); @@ -34,6 +34,7 @@ message empty {} enum TaskState { QUEUED = 0; SOLVED = 1; + LEASED = 2; } message TaskPageRequest { string namespace = 1; diff --git a/engine/src/bin/server.rs b/engine/src/bin/server.rs index 049c97d..c2b8177 100644 --- a/engine/src/bin/server.rs +++ b/engine/src/bin/server.rs @@ -1,15 +1,13 @@ use engine::{get_auth, get_uid}; -use enginelib::api::ServerAPI; use enginelib::plugin::LibraryMetadata; -use enginelib::task::LeasedTask; use enginelib::{ Identifier, RawIdentifier, Registry, - api::EngineAPI, + api::ServerAPI, chrono::Utc, event::{debug, info, warn}, events::{self, Events, ID}, plugin::LibraryManager, - task::{SolvedTasks, StoredExecutingTask, StoredTask, Task, TaskQueue}, + task::{LeasedTask, StoredTask, StoredTaskBlock, Task, TaskQueue}, }; use proto::{ TaskState, @@ -34,7 +32,7 @@ mod proto { } #[allow(non_snake_case)] struct EngineService { - pub EngineAPI: Arc>, + pub EngineAPI: Arc>, } #[tonic::async_trait] impl Engine for EngineService { @@ -72,9 +70,9 @@ impl Engine for EngineService { request: tonic::Request, ) -> Result, Status> { let challenge = get_auth(&request); - let mut api = self.EngineAPI.read().await; + let api = self.EngineAPI.read().await; let db = api.db.clone(); - let output = Events::CheckAdminAuth(&mut api, challenge, ("".into(), "".into()), db); + let output = Events::CheckAdminAuth(&api, challenge, ("".into(), "".into()), db); if !output { warn!("Auth check failed - permission denied"); return Err(tonic::Status::permission_denied("Invalid Auth")); @@ -85,98 +83,44 @@ impl Engine for EngineService { &self, request: tonic::Request, ) -> Result, Status> { - let mut api = self.EngineAPI.write().await; + let api = self.EngineAPI.read().await; let data = request.get_ref(); let challenge = get_auth(&request); let db = api.db.clone(); - let id = ID(&data.namespace, &data.task); + let key = ID(&data.namespace, &data.task); - let output = Events::CheckAdminAuth(&mut api, challenge, ("".into(), "".into()), db); - if !output { + if !Events::CheckAdminAuth(&api, challenge, ("".into(), "".into()), db) { warn!("Auth check failed - permission denied"); return Err(tonic::Status::permission_denied("Invalid Auth")); }; - // Generic helper for removing a task by id from a collection, using an id extractor closure - fn delete_task_from_collection( - collection: &mut HashMap<(String, String), Vec>, - id: &(String, String), - task_id: &str, - state_name: &str, - namespace: &str, - task: &str, - id_extractor: F, - ) -> Result<(), Status> - where - F: Fn(&T) -> &str, - { - match collection.get_mut(id) { - Some(query) => { - let orig_len = query.len(); - query.retain(|f| id_extractor(f) != task_id); - if query.len() == orig_len { - info!( - "DeleteTask: Task with id {} not found in {} state for namespace: {}, task: {}", - task_id, state_name, namespace, task - ); - return Err(Status::not_found(format!( - "Task with id {} not found in {} state", - task_id, state_name - ))); - } - Ok(()) - } - None => { - info!( - "DeleteTask: No tasks found in {} state for namespace: {}, task: {}", - state_name, namespace, task - ); - Err(Status::not_found(format!( - "No tasks found in {} state for given namespace and task", - state_name - ))) - } - } - } - // Use the helper for each state - let result = match data.state() { - TaskState::Processing => delete_task_from_collection( - &mut api.executing_tasks.tasks, - &id, - &data.id, - "Processing", - &data.namespace, - &data.task, - |f| &f.id, - ), - TaskState::Solved => delete_task_from_collection( - &mut api.solved_tasks.tasks, - &id, - &data.id, - "Solved", - &data.namespace, - &data.task, - |f| &f.id, - ), - TaskState::Queued => delete_task_from_collection( - &mut api.task_queue.tasks, - &id, - &data.id, - "Queued", - &data.namespace, - &data.task, - |f| &f.id, - ), + let removed = match data.state() { + TaskState::Queued => api + .delete_queued(&key, &data.id) + .map_err(|e| Status::internal(format!("sled: {e}")))?, + TaskState::Solved => api + .delete_solved(&key, &data.id) + .map_err(|e| Status::internal(format!("sled: {e}")))?, + TaskState::Leased => { + let mut leased = api.leased_tasks.tasks.entry(key.clone()).or_default(); + let orig = leased.len(); + leased.retain(|l| l.stored_task.id != data.id); + orig != leased.len() + } }; - if let Err(e) = result { - return Err(e); + if !removed { + return Err(Status::not_found(format!( + "Task {} not found in {:?} for {}:{}", + data.id, + data.state(), + data.namespace, + data.task, + ))); } - // Sync running memory into DB - EngineAPI::sync_db(&mut api); info!( - "DeleteTask: Successfully deleted task with id {} in state {:?} for namespace: {}, task: {}", + "DeleteTask: deleted {} in {:?} for {}:{}", data.id, data.state(), data.namespace, @@ -211,116 +155,76 @@ impl Engine for EngineService { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status> { - let mut api = self.EngineAPI.write().await; + let api = self.EngineAPI.read().await; let challenge = get_auth(&request); let db = api.db.clone(); - if !Events::CheckAdminAuth(&mut api, challenge, ("".into(), "".into()), db) { + if !Events::CheckAdminAuth(&api, challenge, ("".into(), "".into()), db) { info!("GetTask denied due to Invalid Auth"); return Err(Status::permission_denied("Invalid authentication")); }; let data = request.get_ref(); + let key = ID(&data.namespace, &data.task); + let task_id_str = format!("{}:{}", data.namespace, data.task); + + let to_proto = |id: String, bytes: Vec| proto::Task { + id, + task_id: task_id_str.clone(), + task_payload: bytes, + payload: Vec::new(), + }; - let q: Vec = match data.clone().state() { - TaskState::Processing => { - match api - .executing_tasks - .tasks - .get(&(data.namespace.clone(), data.task.clone())) - { - Some(tasks) => { - let mut task_refs: Vec<_> = tasks.iter().collect(); - task_refs.sort_by_key(|f| &f.id); - task_refs - .iter() - .map(|f| proto::Task { - id: f.id.clone(), - task_id: format!("{}:{}", data.namespace, data.task), - task_payload: f.bytes.clone(), - payload: Vec::new(), - }) - .collect() - } - None => { - info!( - "Namespace {:?} and task {:?} not found in Processing state", - data.namespace, data.task - ); - Vec::new() - } - } - } + let mut tasks: Vec = match data.state() { TaskState::Queued => { - match api - .task_queue - .tasks - .get(&(data.namespace.clone(), data.task.clone())) - { - Some(tasks) => { - let mut d = tasks.clone(); - d.sort_by_key(|f| f.id.clone()); - d.iter() - .map(|f| proto::Task { - id: f.id.clone(), - task_id: format!("{}:{}", data.namespace, data.task), - task_payload: f.bytes.clone(), - payload: Vec::new(), - }) - .collect() - } - None => { - info!( - "Namespace {:?} and task {:?} not found in Queued state", - data.namespace, data.task - ); - Vec::new() - } - } + let mut v: Vec<_> = api + .scan_queued(&key) + .map(|t| to_proto(t.id, t.bytes)) + .collect(); + v.sort_by(|a, b| a.id.cmp(&b.id)); + v } TaskState::Solved => { - match api - .solved_tasks + let mut v: Vec<_> = api + .scan_solved(&key) + .map(|t| to_proto(t.id, t.bytes)) + .collect(); + v.sort_by(|a, b| a.id.cmp(&b.id)); + v + } + TaskState::Leased => { + let mut v: Vec<_> = api + .leased_tasks .tasks - .get(&(data.namespace.clone(), data.task.clone())) - { - Some(tasks) => { - let mut d = tasks.clone(); - d.sort_by_key(|f| f.id.clone()); - d.iter() - .map(|f| proto::Task { - id: f.id.clone(), - task_id: format!("{}:{}", data.namespace, data.task), - task_payload: f.bytes.clone(), - payload: Vec::new(), - }) + .get(&key) + .map(|leased| { + leased + .iter() + .map(|l| to_proto(l.stored_task.id.clone(), l.stored_task.bytes.clone())) .collect() - } - None => { - info!( - "Namespace {:?} and task {:?} not found in Solved state", - data.namespace, data.task - ); - Vec::new() - } - } + }) + .unwrap_or_default(); + v.sort_by(|a, b| a.id.cmp(&b.id)); + v } }; - let index = data.page * data.page_size as u64; - let end = index + (api.cfg.config_toml.pagination_limit.min(data.page_size) as u64); - let final_vec: Vec<_> = q - .iter() - .skip(index as usize) - .take(data.page_size as usize) - .cloned() - .collect(); - return Ok(tonic::Response::new(proto::TaskPage { + + let page_size = api.cfg.config_toml.pagination_limit.min(data.page_size) as usize; + let start = (data.page as usize).saturating_mul(page_size); + let end = start.saturating_add(page_size).min(tasks.len()); + let final_vec = if start >= tasks.len() { + Vec::new() + } else { + tasks.drain(start..end).collect() + }; + + Ok(tonic::Response::new(proto::TaskPage { namespace: data.namespace.clone(), task: data.task.clone(), page: data.page, - page_size: data.page_size, + page_size: page_size as u32, state: data.state, tasks: final_vec, - })); + })) } /// Handles custom gRPC messages with admin-level authentication. /// @@ -394,11 +298,11 @@ impl Engine for EngineService { let uid = get_uid(&request); let challenge = get_auth(&request); info!("Task registry request received from user: {}", uid); - let mut api = self.EngineAPI.write().await; + let api = self.EngineAPI.read().await; let db = api.db.clone(); debug!("Validating authentication for task registry request"); - if !Events::CheckAuth(&mut api, uid.clone(), challenge, db) { + if !Events::CheckAuth(&api, uid.clone(), challenge, db) { info!( "Task registry request denied - invalid authentication for user: {}", uid @@ -406,10 +310,9 @@ impl Engine for EngineService { return Err(Status::permission_denied("Invalid authentication")); }; let mut tasks: Vec = Vec::new(); - for (k, v) in &api.task_registry.tasks { - let js: Vec = vec![k.0.clone(), k.1.clone()]; - let jstr = js.join(":"); - tasks.push(jstr); + for entry in api.task_registry.tasks.iter() { + let k = entry.key(); + tasks.push(format!("{}:{}", k.0, k.1)); } info!("Returning task registry with {} tasks", tasks.len()); let response = proto::TaskRegistry { tasks }; @@ -514,219 +417,211 @@ impl Engine for EngineService { } async fn publish_task_block( &self, - request: tonic::Request, + request: tonic::Request, ) -> Result, tonic::Status> { let challenge = get_auth(&request); let uid = get_uid(&request); - - let task_id = request.get_ref().task_id.clone(); - let instance_id = request.get_ref().id.clone(); - let payload_for_event = Arc::new(std::sync::RwLock::new( - request.get_ref().task_payload.clone(), - )); + let api = self.EngineAPI.read().await; { - let api = self.EngineAPI.read().await; let db = api.db.clone(); if !Events::CheckAuth(&api, uid.clone(), challenge, db) { - info!("Aquire Task denied due to Invalid Auth"); return Err(Status::permission_denied("Invalid authentication")); }; } - { - let api = self.EngineAPI.read().await; - if Events::ServerBeforeTaskPublish( - &api, - uid.clone(), - task_id.clone(), - instance_id.clone(), - payload_for_event.clone(), - ) { - info!( - "ServerBeforeTaskPublish cancelled for user: {} task: {}", - uid, task_id - ); - return Err(Status::aborted( - "Task publish cancelled by server event handler", - )); - } + // Group incoming tasks by their (namespace:task) — the proto allows mixing + // but in practice they'll be uniform per call. Grouping defensively also + // keeps registry lookups + event firing once-per-group. + let mut groups: HashMap> = HashMap::new(); + for t in request.into_inner().tasks { + let Some((ns, name)) = t.task_id.split_once(':') else { + info!("publish: skipping malformed task_id {}", t.task_id); + continue; + }; + groups + .entry((ns.to_string(), name.to_string())) + .or_default() + .push(t); } - let publish_payload = payload_for_event - .read() - .map(|p| p.clone()) - .map_err(|_| Status::internal("Task publish payload lock poisoned"))?; - - let (namespace, task_name) = task_id - .split_once(':') - .ok_or_else(|| Status::invalid_argument("Invalid Params"))?; - let key = ID(namespace, task_name); - - let reg_tsk = { - let api = self.EngineAPI.read().await; - if !api.task_registry.tasks.contains_key(&key) { - warn!( - "Task acquisition failed - task does not exist: {}:{}", - namespace, task_name - ); - return Err(Status::invalid_argument("Task Does not Exist")); - } + for (key, tasks) in groups { + let task_id_str = format!("{}:{}", key.0, key.1); + let Some(reg_tsk) = api.task_registry.get(&key) else { + info!("publish: unknown task {}:{}, skipping group", key.0, key.1); + continue; + }; - match api.task_registry.get(&key) { - Some(r) => r, - None => { - warn!("Task registry missing for {}:{}", namespace, task_name); - return Err(Status::invalid_argument("Task Does not Exist")); + let mut published_ids: Vec = Vec::with_capacity(tasks.len()); + + for t in tasks { + let payload_for_event = + Arc::new(std::sync::RwLock::new(t.task_payload.clone())); + if Events::ServerBeforeTaskPublish( + &api, + uid.clone(), + task_id_str.clone(), + t.id.clone(), + payload_for_event.clone(), + ) { + info!("publish: handler cancelled {}:{}", task_id_str, t.id); + continue; } - } - }; - - if !reg_tsk.verify(publish_payload.clone()) { - info!("Failed to parse task"); - return Err(Status::invalid_argument("Failed to parse given task bytes")); - } - let (solved_id, exec_key_state, solved_key_state, db) = { - let mut api = self.EngineAPI.write().await; + let payload = match payload_for_event.read() { + Ok(p) => p.clone(), + Err(_) => { + info!("publish: payload lock poisoned for {}", t.id); + continue; + } + }; - let solved_id = { - let exec_tasks = api - .executing_tasks - .tasks - .get_mut(&key) - .ok_or_else(|| tonic::Status::not_found("Invalid taskid or userid"))?; + if !reg_tsk.clone().verify(payload.clone()) { + info!("publish: verify failed for {}", t.id); + continue; + } - let idx = exec_tasks + let mut leased = api.leased_tasks.tasks.entry(key.clone()).or_default(); + let Some(idx) = leased .iter() - .position(|f| f.id == instance_id && f.user_id == uid) - .ok_or_else(|| tonic::Status::not_found("Invalid taskid or userid"))?; - - exec_tasks.remove(idx).id - }; - - api.solved_tasks.tasks.entry(key.clone()).or_default().push( - enginelib::task::StoredTask { - bytes: publish_payload.clone(), - id: solved_id.clone(), - }, - ); - - let exec_key_state = api - .executing_tasks - .tasks - .get(&key) - .cloned() - .unwrap_or_default(); - let solved_key_state = api - .solved_tasks - .tasks - .get(&key) - .cloned() - .unwrap_or_default(); - let db = api.db.clone(); - - (solved_id, exec_key_state, solved_key_state, db) - }; - - let exec_op = EngineAPI::state_op_executing(&key, &exec_key_state) - .map_err(|e| Status::internal(format!("Serialization error: {}", e)))?; - let solved_op = EngineAPI::state_op_solved(&key, &solved_key_state) - .map_err(|e| Status::internal(format!("Serialization error: {}", e)))?; - - EngineAPI::apply_batch_ops(&db, vec![exec_op, solved_op]) - .map_err(|e| Status::internal(format!("DB insert error: {}", e)))?; + .position(|l| l.stored_task.id == t.id && l.user_id == uid) + else { + info!("publish: no lease for {} held by {}", t.id, uid); + continue; + }; + leased.remove(idx); + drop(leased); + + let stored = StoredTask { + id: t.id.clone(), + bytes: payload, + }; + if let Err(e) = api.put_solved(&key, &stored) { + info!("publish: sled put_solved failed for {}: {}", t.id, e); + continue; + } + published_ids.push(t.id); + } - { - let api = self.EngineAPI.read().await; - Events::ServerTaskBlockPublished( - &api, - uid.clone(), - task_id.clone(), - vec![solved_id], - ); + if !published_ids.is_empty() { + Events::ServerTaskBlockPublished( + &api, + uid.clone(), + task_id_str, + published_ids, + ); + } } - info!("Task published successfully: {} by user: {}", task_id, uid); Ok(tonic::Response::new(proto::Empty {})) } async fn create_task_block( &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let api = self.EngineAPI.read().await; + request: tonic::Request, + ) -> Result, tonic::Status> { let challenge = get_auth(&request); let uid = get_uid(&request); + let api = self.EngineAPI.read().await; let db = api.db.clone(); if !Events::CheckAuth(&api, uid, challenge, db) { //TODO: change to AdminSpecific Auth info!("Create Task denied due to Invalid Auth"); return Err(Status::permission_denied("Invalid authentication")); }; - let task = request.get_ref(); - let task_id = task.task_id.clone(); - let payload_for_event = Arc::new(std::sync::RwLock::new(task.task_payload.clone())); - if Events::ServerBeforeTaskCreate(&api, task_id.clone(), payload_for_event.clone()) { - info!("ServerBeforeTaskCreate cancelled for task: {}", task_id); - return Err(Status::aborted( - "Task create cancelled by server event handler", - )); - } - let task_payload = payload_for_event - .read() - .map(|p| p.clone()) - .map_err(|_| Status::internal("Task create payload lock poisoned"))?; - - let parts: Vec<&str> = task_id.splitn(2, ':').collect(); - if parts.len() != 2 || parts[0].is_empty() || parts[1].is_empty() { - return Err(Status::invalid_argument( - "Invalid task ID format, expected 'namespace:task'", - )); - } - let id: Identifier = (parts[0].to_string(), parts[1].to_string()); - let tsk_reg = api.task_registry.get(&id); - if let Some(tsk_reg) = tsk_reg { - if !tsk_reg.clone().verify(task_payload.clone()) { - warn!("Failed to parse given task bytes"); - return Err(Status::invalid_argument("Failed to parse given task bytes")); - } - let tbp_tsk = StoredTask { - bytes: task_payload.clone(), - id: druid::Druid::default().to_hex(), + + // Group incoming tasks by (namespace:task). + let mut groups: HashMap> = HashMap::new(); + for t in request.into_inner().tasks { + let Some((ns, name)) = t.task_id.split_once(':') else { + return Err(Status::invalid_argument(format!( + "Invalid task ID format: {}", + t.task_id + ))); }; - api.task_queue - .tasks - .entry(id.clone()) + if ns.is_empty() || name.is_empty() { + return Err(Status::invalid_argument( + "Invalid task ID format, expected 'namespace:task'", + )); + } + groups + .entry((ns.to_string(), name.to_string())) .or_default() - .push(tbp_tsk.clone()); + .push(t); + } + + let mut created: Vec = Vec::new(); - let tasks_key_state = api.task_queue.tasks.get(&id).cloned().unwrap_or_default(); - let task_op = EngineAPI::state_op_tasks(&id, &tasks_key_state) - .map_err(|e| Status::internal(format!("Serialization error: {}", e)))?; - if let Err(e) = EngineAPI::apply_batch_ops(&api.db, vec![task_op]) { - return Err(Status::internal(format!("DB insert error: {}", e))); + for (key, tasks) in groups { + let task_id_str = format!("{}:{}", key.0, key.1); + let Some(reg_tsk) = api.task_registry.get(&key) else { + return Err(Status::invalid_argument(format!( + "Task does not exist: {}", + task_id_str + ))); + }; + + let mut block_tasks: Vec = Vec::with_capacity(tasks.len()); + let mut instance_ids: Vec = Vec::with_capacity(tasks.len()); + let mut payloads: Vec>>> = + Vec::with_capacity(tasks.len()); + + for t in tasks { + let payload_for_event = + Arc::new(std::sync::RwLock::new(t.task_payload.clone())); + if Events::ServerBeforeTaskCreate( + &api, + task_id_str.clone(), + payload_for_event.clone(), + ) { + info!("create: handler cancelled task in {}", task_id_str); + continue; + } + let payload = match payload_for_event.read() { + Ok(p) => p.clone(), + Err(_) => { + info!("create: payload lock poisoned in {}", task_id_str); + continue; + } + }; + if !reg_tsk.clone().verify(payload.clone()) { + info!("create: verify failed in {}", task_id_str); + continue; + } + let stored = StoredTask { + id: druid::Druid::default().to_hex(), + bytes: payload, + }; + if let Err(e) = api.put_queued(&key, &stored) { + info!("create: sled put_queued failed for {}: {}", stored.id, e); + continue; + } + instance_ids.push(stored.id.clone()); + payloads.push(payload_for_event); + created.push(proto::Task { + id: stored.id.clone(), + task_id: task_id_str.clone(), + task_payload: stored.bytes.clone(), + payload: Vec::new(), + }); + block_tasks.push(stored); + } + + if !block_tasks.is_empty() { + if let Some(channel) = api.task_queue.tasks.get(&key) { + let _ = channel.1.try_send(StoredTaskBlock { tasks: block_tasks }); + } + Events::ServerTaskBlockCreated(&api, task_id_str, instance_ids, payloads); } - Events::ServerTaskBlockCreated( - &api, - task_id.clone(), - vec![tbp_tsk.id.clone()], - vec![Arc::new(std::sync::RwLock::new(tbp_tsk.bytes.clone()))], - ); - return Ok(tonic::Response::new(proto::Task { - id: tbp_tsk.id.clone(), - task_id: task_id.clone(), - payload: Vec::new(), - task_payload: tbp_tsk.bytes.clone(), - })); } - Err(tonic::Status::aborted("Error")) + + Ok(tonic::Response::new(proto::TaskBlock { tasks: created })) } } #[tokio::main(flavor = "multi_thread", worker_threads = 8)] async fn main() -> Result<(), Box> { - let mut api = EngineAPI::default(); - EngineAPI::init(&mut api); + let mut api = ServerAPI::default(); + ServerAPI::init(&mut api); Events::init_auth(&mut api); Events::StartEvent(&mut api); Events::ServerStart(&api); @@ -740,7 +635,7 @@ async fn main() -> Result<(), Box> { 50051, ))); let apii = Arc::new(RwLock::new(api)); - EngineAPI::init_chron(apii.clone()); + ServerAPI::init_chron(apii.clone()); let engine = EngineService { EngineAPI: apii }; // Build reflection service, mapping its concrete error into Box diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index 6b8e726..e3fb334 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -104,19 +104,67 @@ impl ServerAPI { spawn(clear_sled_periodically(api, t)); } // type:namespace:task:id - const TASKS_PREFIX: &'static str = "tasks:"; - const SOLVED_PREFIX: &'static str = "solved:"; + pub const TASKS_PREFIX: &'static str = "tasks:"; + pub const SOLVED_PREFIX: &'static str = "solved:"; - fn state_key(prefix: &str, task_id: &Identifier, id: String) -> Vec { + fn state_key(prefix: &str, task_id: &Identifier, id: &str) -> Vec { format!("{}{}\u{1f}{}:{}", prefix, task_id.0, task_id.1, id).into_bytes() } - fn parse_state_key(prefix: &str, key: &[u8]) -> Option { - let key = std::str::from_utf8(key).ok()?; - let rest = key.strip_prefix(prefix)?; - let (task_id, id) = rest.split_once(":")?; - let (namespace, task) = task_id.split_once('\u{1f}')?; - Some((namespace.to_string(), task.to_string())) + fn state_prefix(prefix: &str, task_id: &Identifier) -> Vec { + format!("{}{}\u{1f}{}:", prefix, task_id.0, task_id.1).into_bytes() + } + + pub fn task_key(task_id: &Identifier, id: &str) -> Vec { + Self::state_key(Self::TASKS_PREFIX, task_id, id) + } + + pub fn solved_key(task_id: &Identifier, id: &str) -> Vec { + Self::state_key(Self::SOLVED_PREFIX, task_id, id) + } + + pub fn task_prefix(task_id: &Identifier) -> Vec { + Self::state_prefix(Self::TASKS_PREFIX, task_id) + } + + pub fn solved_prefix(task_id: &Identifier) -> Vec { + Self::state_prefix(Self::SOLVED_PREFIX, task_id) + } + + pub fn put_queued(&self, task_id: &Identifier, task: &StoredTask) -> sled::Result<()> { + let bytes = postcard::to_allocvec(task) + .map_err(|e| sled::Error::Unsupported(format!("postcard: {e}")))?; + self.db.insert(Self::task_key(task_id, &task.id), bytes)?; + Ok(()) + } + + pub fn put_solved(&self, task_id: &Identifier, task: &StoredTask) -> sled::Result<()> { + let bytes = postcard::to_allocvec(task) + .map_err(|e| sled::Error::Unsupported(format!("postcard: {e}")))?; + self.db.insert(Self::solved_key(task_id, &task.id), bytes)?; + Ok(()) + } + + pub fn delete_queued(&self, task_id: &Identifier, id: &str) -> sled::Result { + Ok(self.db.remove(Self::task_key(task_id, id))?.is_some()) + } + + pub fn delete_solved(&self, task_id: &Identifier, id: &str) -> sled::Result { + Ok(self.db.remove(Self::solved_key(task_id, id))?.is_some()) + } + + pub fn scan_queued(&self, task_id: &Identifier) -> impl Iterator + '_ { + self.db + .scan_prefix(Self::task_prefix(task_id)) + .filter_map(|item| item.ok()) + .filter_map(|(_, value)| postcard::from_bytes::(&value).ok()) + } + + pub fn scan_solved(&self, task_id: &Identifier) -> impl Iterator + '_ { + self.db + .scan_prefix(Self::solved_prefix(task_id)) + .filter_map(|item| item.ok()) + .filter_map(|(_, value)| postcard::from_bytes::(&value).ok()) } pub fn fill_queue(api: &ServerAPI, task_id: Identifier) { @@ -143,11 +191,9 @@ impl ServerAPI { .map(|v| v.iter().map(|l| l.stored_task.id.clone()).collect()) .unwrap_or_default(); - // Narrower prefix: only this task's records, not every TASKS_PREFIX row. - let prefix = format!("{}{}\u{1f}{}:", Self::TASKS_PREFIX, task_id.0, task_id.1); let mut block: Vec = Vec::with_capacity(max_block); - for item in api.db.scan_prefix(prefix.as_bytes()) { + for item in api.db.scan_prefix(Self::task_prefix(&task_id)) { let Ok((_, value)) = item else { continue }; let Ok(task) = postcard::from_bytes::(&value) else { continue; From 4ebb2972dbaffc2a2a751c2c45f46af9632ce177 Mon Sep 17 00:00:00 2001 From: IGN-Styly Date: Mon, 1 Jun 2026 18:57:57 +0100 Subject: [PATCH 15/28] Prevent duplicate task refills and stale lease buildup - Serialize queue refills per identifier to avoid TOCTOU duplicates - Remove queued entries after publish and prune expired leases periodically --- engine/proto/engine.proto | 3 --- engine/src/bin/server.rs | 17 ++++++++++-- enginelib/src/api.rs | 56 +++++++++++++++------------------------ 3 files changed, 36 insertions(+), 40 deletions(-) diff --git a/engine/proto/engine.proto b/engine/proto/engine.proto index 1a2f5c4..6af5c55 100644 --- a/engine/proto/engine.proto +++ b/engine/proto/engine.proto @@ -27,9 +27,6 @@ message TaskSelector { string task = 3; string id = 4; } -message TaskBlockSelector { - repeated TaskBlockSelector selector = 1; -} message empty {} enum TaskState { QUEUED = 0; diff --git a/engine/src/bin/server.rs b/engine/src/bin/server.rs index c2b8177..18295cb 100644 --- a/engine/src/bin/server.rs +++ b/engine/src/bin/server.rs @@ -373,8 +373,18 @@ impl Engine for EngineService { }; if receiver.is_empty() { - let api = self.EngineAPI.read().await; - ServerAPI::fill_queue(&api, key.clone()); + // Serialize concurrent refills for this Identifier — without this, + // two acquires that both see an empty channel will both scan sled + // and push duplicate StoredTaskBlocks (P1 TOCTOU). + let lock_arc = { + let api = self.EngineAPI.read().await; + api.fill_locks.entry(key.clone()).or_default().clone() + }; + let _g = lock_arc.lock().await; + if receiver.is_empty() { + let api = self.EngineAPI.read().await; + ServerAPI::fill_queue(&api, key.clone()); + } } let block = receiver @@ -500,6 +510,9 @@ impl Engine for EngineService { info!("publish: sled put_solved failed for {}: {}", t.id, e); continue; } + if let Err(e) = api.delete_queued(&key, &t.id) { + info!("publish: sled delete_queued failed for {}: {}", t.id, e); + } published_ids.push(t.id); } diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index e3fb334..b556c59 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -28,6 +28,9 @@ pub struct ServerAPI { pub event_bus: EventBus, // RW pub db: sled::Db, // R pub lib_manager: LibraryManager, // RW + // Serializes fill_queue() calls per Identifier so concurrent acquires can't + // both refill an empty channel and double-enqueue the same StoredTaskBlock. + pub fill_locks: DashMap>>, } impl Default for ServerAPI { @@ -44,6 +47,7 @@ impl Default for ServerAPI { }, }, leased_tasks: LeasedTaskQueue::default(), + fill_locks: DashMap::new(), } } } @@ -78,6 +82,7 @@ impl ServerAPI { event_handlers: HashMap::new(), }, }, + fill_locks: DashMap::new(), } } pub fn init(api: &mut Self) { @@ -267,40 +272,21 @@ impl Registry for EngineTaskRegistry { } pub async fn clear_sled_periodically(api: Arc>, n_minutes: u64) { - info!("Sled Cron Job Started"); + info!("Lease GC started ({}m interval)", n_minutes); let mut interval = interval(Duration::from_secs(n_minutes * 60)); - // loop { - // interval.tick().await; - // info!("Purging Unsolved Tasks"); - - // let now = Utc::now().timestamp(); - // let mut rw_api = api.write().await; - - // let mut moved_tasks: Vec<(Identifier, StoredTask)> = Vec::new(); - // let mut touched_exec: HashSet = HashSet::new(); - - // for (id, task_list) in rw_api.executing_tasks.tasks.iter_mut() { - // let before_len = task_list.len(); - // task_list.retain(|info| { - // let age = now - info.given_at.timestamp(); - // if age > 3600 { - // info!("Task {:?} is older than an hour! Moving...", info); - // moved_tasks.push(( - // id.clone(), - // StoredTask { - // id: info.id.clone(), - // bytes: info.bytes.clone(), - // }, - // )); - // false - // } else { - // true - // } - // }); - - // if task_list.len() != before_len { - // touched_exec.insert(id.clone()); - // } - // } - // } + let ttl = chrono::Duration::seconds(3600); + loop { + interval.tick().await; + let api = api.read().await; + let now = Utc::now(); + let mut expired: u64 = 0; + for mut entry in api.leased_tasks.tasks.iter_mut() { + let before = entry.len(); + entry.retain(|l| now.signed_duration_since(l.given_at) < ttl); + expired += (before - entry.len()) as u64; + } + if expired > 0 { + info!("Lease GC: expired {} stale lease(s)", expired); + } + } } From f5ccf87a2806571bcdc752d83ead7fa8dae2f189 Mon Sep 17 00:00:00 2001 From: IGN-Styly Date: Mon, 1 Jun 2026 19:42:48 +0100 Subject: [PATCH 16/28] Adopt block-based task RPCs and add TPS benchmark - Switch client/server/packer paths to block-oriented task acquire, publish, and delete calls - Rename task lifecycle events to block variants and update API types - Add an in-process gRPC TPS benchmark for enqueue/process throughput --- engine/Cargo.toml | 6 + engine/benches/engine_tps.rs | 1 + engine/src/bin/bench.rs | 403 +++++++++++ engine/src/bin/client.rs | 128 ++-- engine/src/bin/packer.rs | 113 +-- engine/src/bin/server.rs | 642 +----------------- engine/src/lib.rs | 570 +++++++++++++++- enginelib/src/api.rs | 50 ++ .../src/events/before_task_acquire_event.rs | 27 - .../events/before_task_block_acquire_event.rs | 30 + .../events/before_task_block_execute_event.rs | 46 ++ .../events/before_task_block_publish_event.rs | 46 ++ .../src/events/before_task_execute_event.rs | 43 -- .../src/events/before_task_publish_event.rs | 43 -- enginelib/src/events/mod.rs | 58 +- enginelib/src/events/task_acquired_event.rs | 33 - .../src/events/task_block_acquired_event.rs | 33 + enginelib/src/prelude.rs | 8 +- 18 files changed, 1366 insertions(+), 914 deletions(-) create mode 100644 engine/benches/engine_tps.rs create mode 100644 engine/src/bin/bench.rs delete mode 100644 enginelib/src/events/before_task_acquire_event.rs create mode 100644 enginelib/src/events/before_task_block_acquire_event.rs create mode 100644 enginelib/src/events/before_task_block_execute_event.rs create mode 100644 enginelib/src/events/before_task_block_publish_event.rs delete mode 100644 enginelib/src/events/before_task_execute_event.rs delete mode 100644 enginelib/src/events/before_task_publish_event.rs delete mode 100644 enginelib/src/events/task_acquired_event.rs create mode 100644 enginelib/src/events/task_block_acquired_event.rs diff --git a/engine/Cargo.toml b/engine/Cargo.toml index 9e82df7..792bac7 100644 --- a/engine/Cargo.toml +++ b/engine/Cargo.toml @@ -26,6 +26,12 @@ tonic = "0.14" tonic-prost = "0.14.5" tonic-reflection = "0.14" + +[[bench]] +name = "engine_tps" +path = "benches/engine_tps.rs" +harness = false + [build-dependencies] tonic-build = "0.14" tonic-prost-build = "0.14" diff --git a/engine/benches/engine_tps.rs b/engine/benches/engine_tps.rs new file mode 100644 index 0000000..96df685 --- /dev/null +++ b/engine/benches/engine_tps.rs @@ -0,0 +1 @@ +include!("../src/bin/bench.rs"); diff --git a/engine/src/bin/bench.rs b/engine/src/bin/bench.rs new file mode 100644 index 0000000..8ca8696 --- /dev/null +++ b/engine/src/bin/bench.rs @@ -0,0 +1,403 @@ +// End-to-end TPS benchmark for the block-based gRPC pipeline. +// +// Spins the engine server in-process on a localhost ephemeral port, registers a +// no-op BenchTask, then runs two phases over real loopback gRPC: +// 1. Enqueue N tasks via CreateTaskBlock in chunks +// 2. Drain via W concurrent workers (acquire_task_block + publish_task_block) +// Prints per-phase elapsed/TPS and a linear extrapolation to 1B tasks. + +use clap::Parser; +use engine::{EngineService, proto}; +use enginelib::{ + Identifier, Registry, + api::ServerAPI, + task::{Task, Verifiable}, +}; +use proto::engine_client; +use std::{ + net::{Ipv4Addr, SocketAddr, SocketAddrV4}, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + time::{Duration, Instant}, +}; +use tokio::{ + net::TcpListener, + sync::{RwLock, watch}, + task::JoinSet, +}; +use tonic::{Request, transport::Server}; + +const BENCH_NS: &str = "bench"; +const BENCH_TASK: &str = "noop"; + +fn parse_nonzero_usize(value: &str) -> Result { + let parsed = value + .parse::() + .map_err(|err| format!("expected positive integer: {err}"))?; + if parsed == 0 { + Err("must be greater than zero".to_string()) + } else { + Ok(parsed) + } +} + +fn parse_nonzero_u32(value: &str) -> Result { + let parsed = value + .parse::() + .map_err(|err| format!("expected positive integer: {err}"))?; + if parsed == 0 { + Err("must be greater than zero".to_string()) + } else { + Ok(parsed) + } +} + +fn parse_nonzero_u64(value: &str) -> Result { + let parsed = value + .parse::() + .map_err(|err| format!("expected positive integer: {err}"))?; + if parsed == 0 { + Err("must be greater than zero".to_string()) + } else { + Ok(parsed) + } +} + +#[derive(Parser, Debug)] +#[command(name = "bench", about = "Engine block-based TPS bench")] +struct Args { + /// Total tasks to enqueue + process. + #[arg(long, default_value_t = 10_000_000, value_parser = parse_nonzero_usize)] + tasks: usize, + /// Worker count for the process phase. + #[arg(long, default_value_t = num_cpus_fallback(), value_parser = parse_nonzero_usize)] + workers: usize, + /// Block size for enqueue (CreateTaskBlock chunks). + #[arg(long, default_value_t = 1024, value_parser = parse_nonzero_u32)] + block_size: u32, + /// Payload size in bytes per task (zero-filled). + #[arg(long, default_value_t = 16)] + payload_bytes: usize, + /// Per-acquire timeout so workers can recover if the queue unexpectedly stalls. + #[arg(long, default_value_t = 1000, value_parser = parse_nonzero_u64)] + acquire_timeout_ms: u64, + /// Cargo passes this to harness-free benchmark targets. + #[arg(long = "bench", hide = true, action = clap::ArgAction::SetTrue)] + _cargo_bench: bool, +} + +fn num_cpus_fallback() -> usize { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4) +} + +#[derive(Debug, Clone, Default)] +struct BenchTask; + +impl Verifiable for BenchTask { + fn verify(&self, _b: Vec) -> bool { + true + } +} + +impl Task for BenchTask { + fn get_id(&self) -> Identifier { + (BENCH_NS.to_string(), BENCH_TASK.to_string()) + } + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } + fn run_hip(&mut self) {} + fn run_cpu(&mut self) {} + fn to_bytes(&self) -> Vec { + Vec::new() + } + fn from_bytes(&self, _bytes: &[u8]) -> Box { + Box::new(self.clone()) + } + fn from_toml(&self, _d: String) -> Box { + Box::new(self.clone()) + } + fn to_toml(&self) -> String { + String::new() + } +} + +/// Pick a free localhost port via an ephemeral TcpListener bind+drop. +async fn pick_port() -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + listener.local_addr().unwrap().port() +} + +async fn build_server_api() -> Arc> { + let mut api = ServerAPI::test_default(); + let id: Identifier = (BENCH_NS.to_string(), BENCH_TASK.to_string()); + api.task_registry.register(Arc::new(BenchTask), id.clone()); + api.ensure_task_channel(id); + // Wire up inventory event handlers — the default core::auth_event handler + // approves auth, which the bench client relies on (it sends no creds). + enginelib::event::register_inventory_handlers(&mut api); + Arc::new(RwLock::new(api)) +} + +async fn spawn_server(port: u16) -> Arc> { + let api = build_server_api().await; + let engine = EngineService::new(api.clone()); + let addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), port)); + tokio::spawn(async move { + Server::builder() + .add_service(engine.into_server()) + .serve(addr) + .await + .ok(); + }); + api +} + +async fn build_client(port: u16) -> engine_client::EngineClient { + let url = format!("http://127.0.0.1:{}", port); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let endpoint = tonic::transport::Endpoint::try_from(url.clone()) + .unwrap() + .tcp_nodelay(true); + match endpoint.connect().await { + Ok(channel) => return engine_client::EngineClient::new(channel), + Err(err) if Instant::now() < deadline => { + tokio::time::sleep(Duration::from_millis(20)).await; + let _ = err; + } + Err(err) => panic!("failed to connect to benchmark server on {url}: {err}"), + } + } +} + +fn fmt_dur(d: Duration) -> String { + let secs = d.as_secs_f64(); + if secs >= 3600.0 { + format!("{:.2}h", secs / 3600.0) + } else if secs >= 60.0 { + format!("{:.2}m", secs / 60.0) + } else { + format!("{:.2}s", secs) + } +} + +fn fmt_tps(tps: f64) -> String { + if tps >= 1_000_000.0 { + format!("{:.2}M/s", tps / 1_000_000.0) + } else if tps >= 1_000.0 { + format!("{:.1}k/s", tps / 1_000.0) + } else { + format!("{:.0}/s", tps) + } +} + +async fn enqueue_phase( + client: &mut engine_client::EngineClient, + total: usize, + block_size: u32, + payload_bytes: usize, +) -> Duration { + let task_id = format!("{}:{}", BENCH_NS, BENCH_TASK); + let start = Instant::now(); + let mut sent = 0usize; + while sent < total { + let chunk = (block_size as usize).min(total - sent); + let tasks: Vec = (0..chunk) + .map(|_| proto::Task { + id: String::new(), + task_id: task_id.clone(), + task_payload: vec![0u8; payload_bytes], + payload: Vec::new(), + }) + .collect(); + client + .create_task_block(Request::new(proto::TaskBlock { tasks })) + .await + .expect("create_task_block failed"); + sent += chunk; + } + start.elapsed() +} + +async fn process_phase( + port: u16, + total: usize, + workers: usize, + block_size: u32, + acquire_timeout: Duration, +) -> Result { + let task_id = format!("{}:{}", BENCH_NS, BENCH_TASK); + let counter = Arc::new(AtomicUsize::new(0)); + let (done_tx, done_rx) = watch::channel(false); + let start = Instant::now(); + + let mut handles = JoinSet::new(); + for _ in 0..workers { + let task_id = task_id.clone(); + let counter = counter.clone(); + let done_tx = done_tx.clone(); + let mut done_rx = done_rx.clone(); + handles.spawn(async move { + let mut client = build_client(port).await; + loop { + if *done_rx.borrow() || counter.load(Ordering::Relaxed) >= total { + break; + } + // The server-side acquire waits on a queue receive. The timeout + // catches unexpected stalls, while the finished watch signal + // keeps normal completion from paying the full timeout tail. + let acquire = client.aquire_task_block(Request::new(proto::TaskBlockRequest { + task_id: task_id.clone(), + block_size, + })); + let resp = tokio::select! { + changed = done_rx.changed() => { + if changed.is_ok() && *done_rx.borrow() { + break; + } + continue; + } + resp = tokio::time::timeout(acquire_timeout, acquire) => resp, + }; + let block = match resp { + Ok(Ok(r)) => r.into_inner(), + Ok(Err(status)) => { + if counter.load(Ordering::Relaxed) >= total { + break; + } + if matches!( + status.code(), + tonic::Code::InvalidArgument | tonic::Code::PermissionDenied + ) { + return Err(format!("acquire failed for {task_id}: {status}")); + } + continue; + } + Err(_) => { + if counter.load(Ordering::Relaxed) >= total { + break; + } + continue; + } + }; + if block.tasks.is_empty() { + continue; + } + let n = block.tasks.len(); + // Echo the block right back as the publish. BenchTask::run_hip is + // a no-op so we're measuring pure RPC + storage overhead. + let solved: Vec = block.tasks; + match client + .publish_task_block(Request::new(proto::TaskBlock { tasks: solved })) + .await + { + Ok(_) => { + let processed = counter.fetch_add(n, Ordering::Relaxed) + n; + if processed >= total { + let _ = done_tx.send(true); + break; + } + } + Err(status) => return Err(format!("publish failed for {task_id}: {status}")), + } + } + Ok::<(), String>(()) + }); + } + + while let Some(result) = handles.join_next().await { + match result { + Ok(Ok(())) => {} + Ok(Err(err)) => { + let _ = done_tx.send(true); + handles.abort_all(); + return Err(err); + } + Err(err) => { + let _ = done_tx.send(true); + handles.abort_all(); + return Err(format!("worker task failed: {err}")); + } + } + } + Ok(start.elapsed()) +} + +#[tokio::main(flavor = "multi_thread")] +async fn main() { + let args = Args::parse(); + + println!( + "engine bench: tasks={} workers={} block_size={} payload_bytes={}", + args.tasks, args.workers, args.block_size, args.payload_bytes + ); + + let port = pick_port().await; + let _server_api = spawn_server(port).await; + let mut client = build_client(port).await; + + println!("Phase 1: enqueue"); + let enq = enqueue_phase(&mut client, args.tasks, args.block_size, args.payload_bytes).await; + let enq_tps = args.tasks as f64 / enq.as_secs_f64(); + println!( + " {} tasks in {} ({})", + args.tasks, + fmt_dur(enq), + fmt_tps(enq_tps) + ); + + println!("Phase 2: process"); + let proc = process_phase( + port, + args.tasks, + args.workers, + args.block_size, + Duration::from_millis(args.acquire_timeout_ms), + ) + .await + .expect("process phase failed"); + let proc_tps = args.tasks as f64 / proc.as_secs_f64(); + println!( + " {} tasks in {} ({})", + args.tasks, + fmt_dur(proc), + fmt_tps(proc_tps) + ); + + let total = enq + proc; + let total_tps = args.tasks as f64 / total.as_secs_f64(); + println!( + "Total wall: {} ({} end-to-end)", + fmt_dur(total), + fmt_tps(total_tps) + ); + + let scale = 1_000_000_000.0 / args.tasks as f64; + println!("\n1B linear extrapolation (naive — scaling caveats below):"); + println!( + " Enqueue: {} @ {}", + fmt_dur(enq.mul_f64(scale)), + fmt_tps(enq_tps) + ); + println!( + " Process: {} @ {}", + fmt_dur(proc.mul_f64(scale)), + fmt_tps(proc_tps) + ); + println!( + " End-to-end: {} @ {}", + fmt_dur(total.mul_f64(scale)), + fmt_tps(total_tps) + ); + + println!("\nCaveats: linear extrapolation assumes:"); + println!(" - sled scales linearly with prefix size (it doesn't past tens of GB)"); + println!(" - leased_tasks DashMap memory is bounded (here it churns; at 1B it spikes)"); + println!(" - task_queue_size soft cap doesn't throttle producers"); + println!(" - loopback gRPC ≈ real network (it's typically 2-5x faster than LAN)"); +} diff --git a/engine/src/bin/client.rs b/engine/src/bin/client.rs index e8bc800..85d5c02 100644 --- a/engine/src/bin/client.rs +++ b/engine/src/bin/client.rs @@ -1,5 +1,5 @@ use enginelib::{ - Registry, api::EngineAPI, event::info, events::Events, plugin::LibraryInstance, prelude::debug, + Registry, api::ServerAPI, event::info, events::Events, plugin::LibraryInstance, prelude::debug, }; use proto::engine_client; use std::{collections::HashMap, error::Error, sync::Arc}; @@ -15,8 +15,8 @@ pub mod proto { #[tokio::main] async fn main() -> Result<(), Box> { - let mut api = EngineAPI::default_client(); - EngineAPI::init_client(&mut api); + let mut api = ServerAPI::default_client(); + ServerAPI::init_client(&mut api); Events::ClientStart(&api); compute_module(Arc::new(api)).await; @@ -24,7 +24,7 @@ async fn main() -> Result<(), Box> { } fn make_interceptor( - api_for_interceptor: Arc, + api_for_interceptor: Arc, ) -> impl FnMut(Request<()>) -> Result, tonic::Status> + Clone { move |mut req: Request<()>| { let headers = Arc::new(std::sync::RwLock::new(HashMap::::new())); @@ -47,7 +47,7 @@ fn make_interceptor( async fn worker_loop( worker_id: usize, - api: Arc, + api: Arc, channel: tonic::transport::Channel, task_ids: Arc>, ) { @@ -58,20 +58,21 @@ async fn worker_loop( let mut got_any = false; for task_id in task_ids.iter() { - if Events::BeforeTaskAcquire(api.as_ref(), task_id.clone()) { + if Events::BeforeTaskBlockAcquire(api.as_ref(), vec![task_id.clone()]) { continue; } - let task_resp = client - .aquire_task(Request::new(proto::TaskRequest { + let resp = client + .aquire_task_block(Request::new(proto::TaskBlockRequest { task_id: task_id.clone(), + block_size: 0, })) .await; - let mut task_req = match task_resp { - Ok(resp) => { + let block = match resp { + Ok(r) => { got_any = true; - resp + r.into_inner() } Err(status) if status.code() == tonic::Code::NotFound => continue, Err(status) if status.code() == tonic::Code::PermissionDenied => { @@ -91,69 +92,86 @@ async fn worker_loop( } }; - let task_payload = task_req.get_mut(); + if block.tasks.is_empty() { + continue; + } + + let identifier = match task_id.split_once(':') { + Some(v) => (v.0.to_string(), v.1.to_string()), + None => continue, + }; + let task_def = match api.task_registry.get(&identifier) { + Some(t) => t, + None => continue, + }; - let acquired_payload = - Arc::new(std::sync::RwLock::new(task_payload.task_payload.clone())); - Events::TaskAcquired( + let instance_ids: Vec = block.tasks.iter().map(|t| t.id.clone()).collect(); + let acquired_payloads: Vec>>> = block + .tasks + .iter() + .map(|t| Arc::new(std::sync::RwLock::new(t.task_payload.clone()))) + .collect(); + + Events::TaskBlockAcquired( api.as_ref(), task_id.clone(), - task_payload.id.clone(), - acquired_payload.clone(), + instance_ids.clone(), + acquired_payloads.clone(), ); - if let Ok(payload) = acquired_payload.read() { - task_payload.task_payload = payload.clone(); - } - let exec_payload = Arc::new(std::sync::RwLock::new(task_payload.task_payload.clone())); - if Events::BeforeTaskExecute( + if Events::BeforeTaskBlockExecute( api.as_ref(), task_id.clone(), - task_payload.id.clone(), - exec_payload.clone(), + instance_ids.clone(), + acquired_payloads.clone(), ) { continue; } - if let Ok(payload) = exec_payload.read() { - task_payload.task_payload = payload.clone(); - } - - let identifier = match task_id.split_once(':') { - Some(v) => v, - None => continue, - }; - let task = match api - .task_registry - .get(&(identifier.0.to_string(), identifier.1.to_string())) - { - Some(t) => t, - None => continue, - }; - let mut task = task.from_bytes(&task_payload.task_payload); - task.run_hip(); + let mut solved: Vec = Vec::with_capacity(block.tasks.len()); + let mut publish_payload_locks: Vec>>> = + Vec::with_capacity(block.tasks.len()); + for (t, payload_lock) in block.tasks.iter().zip(acquired_payloads.iter()) { + let payload = match payload_lock.read() { + Ok(p) => p.clone(), + Err(_) => continue, + }; + let mut tsk = task_def.clone().from_bytes(&payload); + tsk.run_hip(); + let out_bytes = tsk.to_bytes(); + publish_payload_locks.push(Arc::new(std::sync::RwLock::new(out_bytes.clone()))); + solved.push(proto::Task { + id: t.id.clone(), + task_id: task_id.clone(), + task_payload: out_bytes, + payload: Vec::new(), + }); + } - let mut solv_task = proto::Task { - id: task_payload.id.clone(), - payload: Vec::new(), - task_id: task_id.clone(), - task_payload: task.to_bytes(), - }; + if solved.is_empty() { + continue; + } - let publish_payload = Arc::new(std::sync::RwLock::new(solv_task.task_payload.clone())); - if Events::BeforeTaskPublish( + if Events::BeforeTaskBlockPublish( api.as_ref(), task_id.clone(), - task_payload.id.clone(), - publish_payload.clone(), + instance_ids, + publish_payload_locks.clone(), ) { continue; } - if let Ok(payload) = publish_payload.read() { - solv_task.task_payload = payload.clone(); + + // Sync any handler-modified payloads back into the outgoing block. + for (s, lock) in solved.iter_mut().zip(publish_payload_locks.iter()) { + if let Ok(p) = lock.read() { + s.task_payload = p.clone(); + } } - if let Err(status) = client.publish_task(Request::new(solv_task)).await { + if let Err(status) = client + .publish_task_block(Request::new(proto::TaskBlock { tasks: solved })) + .await + { debug!( "worker {}: publish failed for {}: {:?}", worker_id, task_id, status @@ -170,7 +188,7 @@ async fn worker_loop( // Compute Module // Verifies server and also is // Responsible for getting task, executing and publishing it. -async fn compute_module(api: Arc) { +async fn compute_module(api: Arc) { let url = "http://[::1]:50051"; let endpoint = Endpoint::from_static(url) .tcp_nodelay(true) diff --git a/engine/src/bin/packer.rs b/engine/src/bin/packer.rs index 46cf05c..ba37461 100644 --- a/engine/src/bin/packer.rs +++ b/engine/src/bin/packer.rs @@ -1,18 +1,15 @@ use clap::{Args, CommandFactory, Subcommand, ValueEnum, ValueHint}; use clap::{Command, Parser}; use clap_complete::{Generator, Shell, generate}; -use colored::*; -use enginelib::events::{Events, ID}; -// For coloring the output use enginelib::Registry; use enginelib::api::postcard; +use enginelib::events::{Events, ID}; use enginelib::prelude::error; -use enginelib::task::{StoredTask, Task, TaskQueue}; -use enginelib::{api::EngineAPI, config::Config, event::info}; -use serde::Deserialize; +use enginelib::task::StoredTask; +use enginelib::{api::ServerAPI, config::Config, event::info}; +use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::collections::HashMap; -use std::ffi::OsString; use std::fs::File; use std::io::Write; use std::io::{self, BufReader, ErrorKind, Read}; @@ -114,7 +111,7 @@ impl StateArg { fn to_proto(&self) -> i32 { match self { StateArg::Queued => proto::TaskState::Queued as i32, - StateArg::Processing => proto::TaskState::Processing as i32, + StateArg::Processing => proto::TaskState::Leased as i32, StateArg::Solved => proto::TaskState::Solved as i32, } } @@ -174,7 +171,7 @@ fn print_completions(generator: G, cmd: &mut Command) { ); } -fn build_headers(api: &EngineAPI, admin: bool) -> HashMap { +fn build_headers(api: &ServerAPI, admin: bool) -> HashMap { let headers = std::sync::Arc::new(std::sync::RwLock::new(HashMap::::new())); Events::ClientAuthPrepare(api, headers.clone()); let mut prepared_headers = headers.read().map(|h| h.clone()).unwrap_or_default(); @@ -201,6 +198,17 @@ struct ChunkedTaskRecord { payload: Vec, } +#[derive(Debug, Default, Serialize, Deserialize)] +struct PackedTaskQueue { + tasks: BTreeMap<(String, String), Vec>, +} + +impl PackedTaskQueue { + fn push(&mut self, key: (String, String), task: StoredTask) { + self.tasks.entry(key).or_default().push(task); + } +} + fn write_chunked_header(writer: &mut W) -> io::Result<()> { writer.write_all(&CHUNKED_MAGIC)?; writer.write_all(&[CHUNKED_VERSION])?; @@ -353,13 +361,10 @@ async fn main() { eprintln!("Generating completion file for {generator:?}..."); print_completions(generator, &mut cmd); } - let mut api = EngineAPI::default_client(); - EngineAPI::init_client(&mut api); + let mut api = ServerAPI::default_client(); + ServerAPI::init_client(&mut api); // packer intentionally uses client init path; load config explicitly for host/admin token api.cfg = Config::new(); - for (id, tsk) in api.task_registry.tasks.iter() { - api.task_queue.tasks.entry(id.clone()).or_default(); - } if let Some(command) = cli.command { match command { Commands::Schema => { @@ -411,8 +416,8 @@ async fn main() { // Try to deserialize. Only on successful deserialization do we // process entries and write the output TOML file. - let maybe_queue: Option = - match postcard::from_bytes::(&buf) { + let maybe_queue: Option = + match postcard::from_bytes::(&buf) { Ok(k) => Some(k), Err(e) => { error!("Failed to deserialize task queue: {}", e); @@ -546,8 +551,11 @@ async fn main() { payload: Vec::new(), }; - match client.create_task(Request::new(req)).await { - Ok(_) => uploaded += 1, + match client + .create_task_block(Request::new(proto::TaskBlock { tasks: vec![req] })) + .await + { + Ok(resp) => uploaded += resp.into_inner().tasks.len(), Err(e) => { failed += 1; error!("Failed to upload task {}: {}", task_id, e); @@ -573,7 +581,8 @@ async fn main() { } } - let queue: TaskQueue = match postcard::from_bytes::(&buf) { + let queue: PackedTaskQueue = match postcard::from_bytes::(&buf) + { Ok(q) => q, Err(e) => { error!("Failed to deserialize task queue: {}", e); @@ -583,19 +592,25 @@ async fn main() { for ((namespace, task), tasks) in queue.tasks { let task_id = format!("{}:{}", namespace, task); - for stored in tasks { - let req = proto::Task { + let requested = tasks.len(); + let requests: Vec = tasks + .into_iter() + .map(|stored| proto::Task { id: stored.id, task_id: task_id.clone(), task_payload: stored.bytes, payload: Vec::new(), - }; - match client.create_task(Request::new(req)).await { - Ok(_) => uploaded += 1, - Err(e) => { - failed += 1; - error!("Failed to upload task {}: {}", task_id, e); - } + }) + .collect(); + + match client + .create_task_block(Request::new(proto::TaskBlock { tasks: requests })) + .await + { + Ok(resp) => uploaded += resp.into_inner().tasks.len(), + Err(e) => { + failed += requested; + error!("Failed to upload task {}: {}", task_id, e); } } } @@ -790,7 +805,7 @@ async fn main() { vec![args.state.clone()] }; - let mut out_queue = TaskQueue::default(); + let mut out_queue = PackedTaskQueue::default(); let mut fetched = 0usize; for task_id in task_ids { @@ -825,13 +840,14 @@ async fn main() { break; } - let key = ID(namespace, task); - let bucket = out_queue.tasks.entry(key).or_default(); for t in resp.tasks { - bucket.push(StoredTask { - bytes: t.task_payload, - id: t.id, - }); + out_queue.push( + ID(namespace, task), + StoredTask { + bytes: t.task_payload, + id: t.id, + }, + ); fetched += 1; } @@ -899,7 +915,7 @@ async fn main() { id: args.id.clone(), }; - match client.delete_task(Request::new(req)).await { + match client.delete_task_block(Request::new(req)).await { Ok(_) => info!( "Deleted task {} from {}:{} ({:?})", args.id, args.namespace, args.task, args.state @@ -915,6 +931,7 @@ async fn main() { match toml::from_str::(&toml_str) { Ok(raw) => { let entries = parse_entries(raw); + let mut out_queue = PackedTaskQueue::default(); for entry in entries { match api .task_registry @@ -928,17 +945,13 @@ async fn main() { entry.namespace.as_str(), entry.id.as_str(), ); - let mut vec = api - .task_queue - .tasks - .get(&key) - .cloned() - .unwrap_or_default(); - vec.push(StoredTask { - id: "".into(), //ids are minted on the server - bytes: t.to_bytes(), - }); - api.task_queue.tasks.insert(key, vec); + out_queue.push( + key, + StoredTask { + id: "".into(), //ids are minted on the server + bytes: t.to_bytes(), + }, + ); } Err(e) => { error!( @@ -968,9 +981,7 @@ async fn main() { } let mut wrote = 0usize; - for ((namespace, task), tasks) in - &api.task_queue.tasks - { + for ((namespace, task), tasks) in &out_queue.tasks { for stored in tasks { let record = ChunkedTaskRecord { namespace: namespace.clone(), @@ -1012,7 +1023,7 @@ async fn main() { } } } else { - match postcard::to_allocvec(&api.task_queue) { + match postcard::to_allocvec(&out_queue) { Ok(data) => { match File::create("output.rustforge.bin") { Ok(mut file) => { diff --git a/engine/src/bin/server.rs b/engine/src/bin/server.rs index 18295cb..09003da 100644 --- a/engine/src/bin/server.rs +++ b/engine/src/bin/server.rs @@ -1,635 +1,11 @@ -use engine::{get_auth, get_uid}; -use enginelib::plugin::LibraryMetadata; -use enginelib::{ - Identifier, RawIdentifier, Registry, - api::ServerAPI, - chrono::Utc, - event::{debug, info, warn}, - events::{self, Events, ID}, - plugin::LibraryManager, - task::{LeasedTask, StoredTask, StoredTaskBlock, Task, TaskQueue}, -}; -use proto::{ - TaskState, - engine_server::{Engine, EngineServer}, -}; +use engine::{EngineService, proto}; +use enginelib::{api::ServerAPI, event::info, events::Events}; use std::{ - collections::HashMap, - env::consts::OS, - io::Read, - net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4}, - sync::{Arc, RwLock as RS_RwLock}, + net::{Ipv4Addr, SocketAddr, SocketAddrV4}, + sync::Arc, }; use tokio::sync::RwLock; -use tonic::{Request, Response, Status, metadata::MetadataValue, transport::Server}; - -use crate::proto::ModuleInfo; - -mod proto { - tonic::include_proto!("engine"); - pub(crate) const FILE_DESCRIPTOR_SET: &[u8] = - tonic::include_file_descriptor_set!("engine_descriptor"); -} -#[allow(non_snake_case)] -struct EngineService { - pub EngineAPI: Arc>, -} -#[tonic::async_trait] -impl Engine for EngineService { - async fn get_metadata( - &self, - request: tonic::Request, - ) -> Result, Status> { - let api = self.EngineAPI.read().await; - - let modules: Vec = api - .lib_manager - .libraries - .values() - .map(|lib| lib.metadata.clone()) - //Dont show server mods - .filter(|lib| !lib.mod_server) - // vec> --> vec - .map(|f| ModuleInfo { - mod_id: f.mod_id.clone(), - api_version: f.api_version.clone(), - rustc_version: f.rustc_version.clone(), - mod_version: f.mod_version.clone(), - }) - .collect(); - - let res = proto::ServerMetadata { - engine_api: enginelib::GIT_VERSION.to_string(), - mods: modules, - }; - return Ok(Response::new(res)); - } - - async fn check_auth( - &self, - request: tonic::Request, - ) -> Result, Status> { - let challenge = get_auth(&request); - let api = self.EngineAPI.read().await; - let db = api.db.clone(); - let output = Events::CheckAdminAuth(&api, challenge, ("".into(), "".into()), db); - if !output { - warn!("Auth check failed - permission denied"); - return Err(tonic::Status::permission_denied("Invalid Auth")); - }; - return Ok(tonic::Response::new(proto::Empty {})); - } - async fn delete_task_block( - &self, - request: tonic::Request, - ) -> Result, Status> { - let api = self.EngineAPI.read().await; - let data = request.get_ref(); - let challenge = get_auth(&request); - let db = api.db.clone(); - let key = ID(&data.namespace, &data.task); - - if !Events::CheckAdminAuth(&api, challenge, ("".into(), "".into()), db) { - warn!("Auth check failed - permission denied"); - return Err(tonic::Status::permission_denied("Invalid Auth")); - }; - - let removed = match data.state() { - TaskState::Queued => api - .delete_queued(&key, &data.id) - .map_err(|e| Status::internal(format!("sled: {e}")))?, - TaskState::Solved => api - .delete_solved(&key, &data.id) - .map_err(|e| Status::internal(format!("sled: {e}")))?, - TaskState::Leased => { - let mut leased = api.leased_tasks.tasks.entry(key.clone()).or_default(); - let orig = leased.len(); - leased.retain(|l| l.stored_task.id != data.id); - orig != leased.len() - } - }; - - if !removed { - return Err(Status::not_found(format!( - "Task {} not found in {:?} for {}:{}", - data.id, - data.state(), - data.namespace, - data.task, - ))); - } - - info!( - "DeleteTask: deleted {} in {:?} for {}:{}", - data.id, - data.state(), - data.namespace, - data.task - ); - Ok(tonic::Response::new(proto::Empty {})) - } - /// Retrieves a paginated list of tasks filtered by namespace, task name, and state. - /// - /// Authenticates the request and, if authorized, returns tasks in the specified state - /// (`Processing`, `Queued`, or `Solved`) for the given namespace and task name. The results - /// are sorted by task ID and paginated according to the requested page and page size. - /// - /// Returns a `TaskPage` containing the filtered tasks and pagination metadata, or a - /// permission denied error if authentication fails. - /// - /// # Examples - /// - /// ``` - /// // Example usage within a tonic gRPC client context: - /// let request = proto::TaskPageRequest { - /// namespace: "example_ns".to_string(), - /// task: "example_task".to_string(), - /// state: proto::TaskState::Queued as i32, - /// page: 0, - /// page_size: 10, - /// }; - /// let response = engine_client.get_tasks(request).await?; - /// assert!(response.get_ref().tasks.len() <= 10); - /// ``` - async fn get_tasks( - &self, - request: tonic::Request, - ) -> std::result::Result, tonic::Status> { - let api = self.EngineAPI.read().await; - let challenge = get_auth(&request); - - let db = api.db.clone(); - if !Events::CheckAdminAuth(&api, challenge, ("".into(), "".into()), db) { - info!("GetTask denied due to Invalid Auth"); - return Err(Status::permission_denied("Invalid authentication")); - }; - let data = request.get_ref(); - let key = ID(&data.namespace, &data.task); - let task_id_str = format!("{}:{}", data.namespace, data.task); - - let to_proto = |id: String, bytes: Vec| proto::Task { - id, - task_id: task_id_str.clone(), - task_payload: bytes, - payload: Vec::new(), - }; - - let mut tasks: Vec = match data.state() { - TaskState::Queued => { - let mut v: Vec<_> = api - .scan_queued(&key) - .map(|t| to_proto(t.id, t.bytes)) - .collect(); - v.sort_by(|a, b| a.id.cmp(&b.id)); - v - } - TaskState::Solved => { - let mut v: Vec<_> = api - .scan_solved(&key) - .map(|t| to_proto(t.id, t.bytes)) - .collect(); - v.sort_by(|a, b| a.id.cmp(&b.id)); - v - } - TaskState::Leased => { - let mut v: Vec<_> = api - .leased_tasks - .tasks - .get(&key) - .map(|leased| { - leased - .iter() - .map(|l| to_proto(l.stored_task.id.clone(), l.stored_task.bytes.clone())) - .collect() - }) - .unwrap_or_default(); - v.sort_by(|a, b| a.id.cmp(&b.id)); - v - } - }; - - let page_size = api.cfg.config_toml.pagination_limit.min(data.page_size) as usize; - let start = (data.page as usize).saturating_mul(page_size); - let end = start.saturating_add(page_size).min(tasks.len()); - let final_vec = if start >= tasks.len() { - Vec::new() - } else { - tasks.drain(start..end).collect() - }; - - Ok(tonic::Response::new(proto::TaskPage { - namespace: data.namespace.clone(), - task: data.task.clone(), - page: data.page, - page_size: page_size as u32, - state: data.state, - tasks: final_vec, - })) - } - /// Handles custom gRPC messages with admin-level authentication. - /// - /// Processes a CGRPC request by verifying admin credentials and dispatching the event payload to the appropriate handler. Returns the processed event payload in the response. If authentication fails, returns a permission denied error. - /// - /// # Returns - /// A `Cgrpcmsg` response containing the processed event payload, or a permission denied gRPC status on failed authentication. - /// - /// # Examples - /// - /// ``` - /// // Example usage within a gRPC client context: - /// let request = proto::Cgrpcmsg { - /// handler_mod_id: "mod".to_string(), - /// handler_id: "handler".to_string(), - /// event_payload: vec![1, 2, 3], - /// // ... other fields ... - /// }; - /// let response = engine_service.cgrpc(tonic::Request::new(request)).await?; - /// assert_eq!(response.get_ref().handler_mod_id, "mod"); - /// ``` - async fn cgrpc( - &self, - request: tonic::Request, - ) -> std::result::Result, tonic::Status> { - info!( - "CGRPC request received for handler: {}:{}", - request.get_ref().handler_mod_id, - request.get_ref().handler_id - ); - let mut api = self.EngineAPI.write().await; - let challenge = get_auth(&request); - let db = api.db.clone(); - debug!("Checking admin authentication for CGRPC request"); - let output = Events::CheckAdminAuth( - &mut api, - challenge, - ( - request.get_ref().handler_mod_id.clone(), - request.get_ref().handler_id.clone(), - ), - db, - ); - if !output { - warn!("CGRPC auth check failed - permission denied"); - return Err(tonic::Status::permission_denied("Invalid CGRPC Auth")); - }; - let out = Arc::new(std::sync::RwLock::new(Vec::new())); - debug!("Dispatching CGRPC event to handler"); - Events::CgrpcEvent( - &mut api, - ID("engine_core", "grpc"), - request.get_ref().event_payload.clone(), - out.clone(), - ); - let mut res = request.get_ref().clone(); - res.event_payload = match out.read() { - Ok(g) => g.clone(), - Err(_) => { - warn!("CGRPC response lock poisoned, returning empty payload"); - Vec::new() - } - }; - info!("CGRPC request processed successfully"); - return Ok(tonic::Response::new(res)); - } - async fn aquire_task_reg( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let uid = get_uid(&request); - let challenge = get_auth(&request); - info!("Task registry request received from user: {}", uid); - let api = self.EngineAPI.read().await; - let db = api.db.clone(); - - debug!("Validating authentication for task registry request"); - if !Events::CheckAuth(&api, uid.clone(), challenge, db) { - info!( - "Task registry request denied - invalid authentication for user: {}", - uid - ); - return Err(Status::permission_denied("Invalid authentication")); - }; - let mut tasks: Vec = Vec::new(); - for entry in api.task_registry.tasks.iter() { - let k = entry.key(); - tasks.push(format!("{}:{}", k.0, k.1)); - } - info!("Returning task registry with {} tasks", tasks.len()); - let response = proto::TaskRegistry { tasks }; - Ok(tonic::Response::new(response)) - } - - async fn aquire_task_block( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let challenge = get_auth(&request); - let task_id = request.get_ref().task_id.clone(); - let uid = get_uid(&request); - info!( - "Task acquisition request received from user: {} for task: {}", - uid, task_id - ); - - { - let api = self.EngineAPI.read().await; - let db = api.db.clone(); - if !Events::CheckAuth(&api, uid.clone(), challenge, db) { - info!( - "Task acquisition denied - invalid authentication for user: {}", - uid - ); - return Err(Status::permission_denied("Invalid authentication")); - }; - } - - let (namespace, task_name) = task_id.split_once(':').ok_or_else(|| { - Status::invalid_argument("Invalid task ID format, expected 'namespace:task'") - })?; - let key = ID(namespace, task_name); - - { - let api = self.EngineAPI.read().await; - if api.task_registry.get(&key).is_none() { - return Err(Status::invalid_argument("Task Does not Exist")); - } - if Events::ServerBeforeTaskAcquire(&api, uid.clone(), task_id.clone()) { - return Err(Status::aborted( - "Task acquire cancelled by server event handler", - )); - } - } - - // Resolve the receiver, refill from sled if drained, then await a block. - // recv blocks until a producer (fill_queue or create_task_block) pushes one, - // so callers should use a gRPC deadline if they need an upper bound. - let receiver = { - let api = self.EngineAPI.read().await; - api.task_queue - .tasks - .get(&key) - .map(|entry| entry.0.clone()) - .ok_or_else(|| Status::not_found("Unknown task type"))? - }; - - if receiver.is_empty() { - // Serialize concurrent refills for this Identifier — without this, - // two acquires that both see an empty channel will both scan sled - // and push duplicate StoredTaskBlocks (P1 TOCTOU). - let lock_arc = { - let api = self.EngineAPI.read().await; - api.fill_locks.entry(key.clone()).or_default().clone() - }; - let _g = lock_arc.lock().await; - if receiver.is_empty() { - let api = self.EngineAPI.read().await; - ServerAPI::fill_queue(&api, key.clone()); - } - } - - let block = receiver - .recv() - .await - .map_err(|_| Status::unavailable("Task queue closed"))?; - - // Lease the block and fire one per-block acquire event. - let instance_ids: Vec = { - let api = self.EngineAPI.read().await; - let mut entry = api.leased_tasks.tasks.entry(key.clone()).or_default(); - let now = Utc::now(); - let mut ids = Vec::with_capacity(block.tasks.len()); - for task in &block.tasks { - ids.push(task.id.clone()); - entry.push(LeasedTask { - stored_task: Arc::new(task.clone()), - user_id: uid.clone(), - given_at: now, - }); - } - drop(entry); - Events::ServerTaskBlockAcquired(&api, uid.clone(), task_id.clone(), ids.clone()); - ids - }; - let _ = instance_ids; - - let tasks = block - .tasks - .into_iter() - .map(|t| proto::Task { - id: t.id, - task_id: task_id.clone(), - task_payload: t.bytes, - payload: Vec::new(), - }) - .collect(); - - Ok(tonic::Response::new(proto::TaskBlock { tasks })) - } - async fn publish_task_block( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let challenge = get_auth(&request); - let uid = get_uid(&request); - let api = self.EngineAPI.read().await; - - { - let db = api.db.clone(); - if !Events::CheckAuth(&api, uid.clone(), challenge, db) { - return Err(Status::permission_denied("Invalid authentication")); - }; - } - - // Group incoming tasks by their (namespace:task) — the proto allows mixing - // but in practice they'll be uniform per call. Grouping defensively also - // keeps registry lookups + event firing once-per-group. - let mut groups: HashMap> = HashMap::new(); - for t in request.into_inner().tasks { - let Some((ns, name)) = t.task_id.split_once(':') else { - info!("publish: skipping malformed task_id {}", t.task_id); - continue; - }; - groups - .entry((ns.to_string(), name.to_string())) - .or_default() - .push(t); - } - - for (key, tasks) in groups { - let task_id_str = format!("{}:{}", key.0, key.1); - let Some(reg_tsk) = api.task_registry.get(&key) else { - info!("publish: unknown task {}:{}, skipping group", key.0, key.1); - continue; - }; - - let mut published_ids: Vec = Vec::with_capacity(tasks.len()); - - for t in tasks { - let payload_for_event = - Arc::new(std::sync::RwLock::new(t.task_payload.clone())); - if Events::ServerBeforeTaskPublish( - &api, - uid.clone(), - task_id_str.clone(), - t.id.clone(), - payload_for_event.clone(), - ) { - info!("publish: handler cancelled {}:{}", task_id_str, t.id); - continue; - } - - let payload = match payload_for_event.read() { - Ok(p) => p.clone(), - Err(_) => { - info!("publish: payload lock poisoned for {}", t.id); - continue; - } - }; - - if !reg_tsk.clone().verify(payload.clone()) { - info!("publish: verify failed for {}", t.id); - continue; - } - - let mut leased = api.leased_tasks.tasks.entry(key.clone()).or_default(); - let Some(idx) = leased - .iter() - .position(|l| l.stored_task.id == t.id && l.user_id == uid) - else { - info!("publish: no lease for {} held by {}", t.id, uid); - continue; - }; - leased.remove(idx); - drop(leased); - - let stored = StoredTask { - id: t.id.clone(), - bytes: payload, - }; - if let Err(e) = api.put_solved(&key, &stored) { - info!("publish: sled put_solved failed for {}: {}", t.id, e); - continue; - } - if let Err(e) = api.delete_queued(&key, &t.id) { - info!("publish: sled delete_queued failed for {}: {}", t.id, e); - } - published_ids.push(t.id); - } - - if !published_ids.is_empty() { - Events::ServerTaskBlockPublished( - &api, - uid.clone(), - task_id_str, - published_ids, - ); - } - } - - Ok(tonic::Response::new(proto::Empty {})) - } - async fn create_task_block( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let challenge = get_auth(&request); - let uid = get_uid(&request); - let api = self.EngineAPI.read().await; - let db = api.db.clone(); - if !Events::CheckAuth(&api, uid, challenge, db) { - //TODO: change to AdminSpecific Auth - info!("Create Task denied due to Invalid Auth"); - return Err(Status::permission_denied("Invalid authentication")); - }; - - // Group incoming tasks by (namespace:task). - let mut groups: HashMap> = HashMap::new(); - for t in request.into_inner().tasks { - let Some((ns, name)) = t.task_id.split_once(':') else { - return Err(Status::invalid_argument(format!( - "Invalid task ID format: {}", - t.task_id - ))); - }; - if ns.is_empty() || name.is_empty() { - return Err(Status::invalid_argument( - "Invalid task ID format, expected 'namespace:task'", - )); - } - groups - .entry((ns.to_string(), name.to_string())) - .or_default() - .push(t); - } - - let mut created: Vec = Vec::new(); - - for (key, tasks) in groups { - let task_id_str = format!("{}:{}", key.0, key.1); - let Some(reg_tsk) = api.task_registry.get(&key) else { - return Err(Status::invalid_argument(format!( - "Task does not exist: {}", - task_id_str - ))); - }; - - let mut block_tasks: Vec = Vec::with_capacity(tasks.len()); - let mut instance_ids: Vec = Vec::with_capacity(tasks.len()); - let mut payloads: Vec>>> = - Vec::with_capacity(tasks.len()); - - for t in tasks { - let payload_for_event = - Arc::new(std::sync::RwLock::new(t.task_payload.clone())); - if Events::ServerBeforeTaskCreate( - &api, - task_id_str.clone(), - payload_for_event.clone(), - ) { - info!("create: handler cancelled task in {}", task_id_str); - continue; - } - let payload = match payload_for_event.read() { - Ok(p) => p.clone(), - Err(_) => { - info!("create: payload lock poisoned in {}", task_id_str); - continue; - } - }; - if !reg_tsk.clone().verify(payload.clone()) { - info!("create: verify failed in {}", task_id_str); - continue; - } - let stored = StoredTask { - id: druid::Druid::default().to_hex(), - bytes: payload, - }; - if let Err(e) = api.put_queued(&key, &stored) { - info!("create: sled put_queued failed for {}: {}", stored.id, e); - continue; - } - instance_ids.push(stored.id.clone()); - payloads.push(payload_for_event); - created.push(proto::Task { - id: stored.id.clone(), - task_id: task_id_str.clone(), - task_payload: stored.bytes.clone(), - payload: Vec::new(), - }); - block_tasks.push(stored); - } - - if !block_tasks.is_empty() { - if let Some(channel) = api.task_queue.tasks.get(&key) { - let _ = channel.1.try_send(StoredTaskBlock { tasks: block_tasks }); - } - Events::ServerTaskBlockCreated(&api, task_id_str, instance_ids, payloads); - } - } - - Ok(tonic::Response::new(proto::TaskBlock { tasks: created })) - } -} +use tonic::transport::Server; #[tokio::main(flavor = "multi_thread", worker_threads = 8)] async fn main() -> Result<(), Box> { @@ -649,18 +25,18 @@ async fn main() -> Result<(), Box> { ))); let apii = Arc::new(RwLock::new(api)); ServerAPI::init_chron(apii.clone()); - let engine = EngineService { EngineAPI: apii }; + let engine = EngineService::new(apii); + + info!("Engine listening on {}", addr); - // Build reflection service, mapping its concrete error into Box let reflection_service = tonic_reflection::server::Builder::configure() .register_encoded_file_descriptor_set(proto::FILE_DESCRIPTOR_SET) .build_v1() .map_err(|e| Box::new(e) as Box)?; - // Start server and map transport errors into Box so `?` works with our return type. Server::builder() .add_service(reflection_service) - .add_service(EngineServer::new(engine)) + .add_service(engine.into_server()) .serve(addr) .await .map_err(|e| Box::new(e) as Box)?; diff --git a/engine/src/lib.rs b/engine/src/lib.rs index 5b529f0..534ac0a 100644 --- a/engine/src/lib.rs +++ b/engine/src/lib.rs @@ -1,4 +1,26 @@ -use tonic::Request; +use enginelib::{ + Identifier, RawIdentifier, Registry, + api::ServerAPI, + chrono::Utc, + event::{debug, info, warn}, + events::{Events, ID}, + task::{LeasedTask, StoredTask, StoredTaskBlock}, +}; +use std::{collections::HashMap, sync::Arc}; +use tokio::sync::RwLock; +use tonic::{Request, Response, Status}; + +pub mod proto { + tonic::include_proto!("engine"); + pub const FILE_DESCRIPTOR_SET: &[u8] = tonic::include_file_descriptor_set!("engine_descriptor"); +} + +use proto::{ + ModuleInfo, TaskState, + engine_server::{Engine, EngineServer}, +}; + +pub use proto::engine_server::EngineServer as EngineGrpcServer; pub fn get_uid(req: &Request) -> String { req.metadata() @@ -15,3 +37,549 @@ pub fn get_auth(req: &Request) -> String { .map(|s| s.to_string()) .unwrap_or_default() } + +#[allow(non_snake_case)] +pub struct EngineService { + pub EngineAPI: Arc>, +} + +impl EngineService { + pub fn new(api: Arc>) -> Self { + Self { EngineAPI: api } + } + + pub fn into_server(self) -> EngineServer { + EngineServer::new(self) + } +} + +#[tonic::async_trait] +impl Engine for EngineService { + async fn get_metadata( + &self, + _request: tonic::Request, + ) -> Result, Status> { + let api = self.EngineAPI.read().await; + + let modules: Vec = api + .lib_manager + .libraries + .values() + .map(|lib| lib.metadata.clone()) + .filter(|lib| !lib.mod_server) + .map(|f| ModuleInfo { + mod_id: f.mod_id.clone(), + api_version: f.api_version.clone(), + rustc_version: f.rustc_version.clone(), + mod_version: f.mod_version.clone(), + }) + .collect(); + + let res = proto::ServerMetadata { + engine_api: enginelib::GIT_VERSION.to_string(), + mods: modules, + }; + Ok(Response::new(res)) + } + + async fn check_auth( + &self, + request: tonic::Request, + ) -> Result, Status> { + let challenge = get_auth(&request); + let api = self.EngineAPI.read().await; + let db = api.db.clone(); + let output = Events::CheckAdminAuth(&api, challenge, ("".into(), "".into()), db); + if !output { + warn!("Auth check failed - permission denied"); + return Err(tonic::Status::permission_denied("Invalid Auth")); + }; + Ok(tonic::Response::new(proto::Empty {})) + } + + async fn delete_task_block( + &self, + request: tonic::Request, + ) -> Result, Status> { + let api = self.EngineAPI.read().await; + let data = request.get_ref(); + let challenge = get_auth(&request); + let db = api.db.clone(); + let key = ID(&data.namespace, &data.task); + + if !Events::CheckAdminAuth(&api, challenge, ("".into(), "".into()), db) { + warn!("Auth check failed - permission denied"); + return Err(tonic::Status::permission_denied("Invalid Auth")); + }; + + let removed = match data.state() { + TaskState::Queued => api + .delete_queued(&key, &data.id) + .map_err(|e| Status::internal(format!("sled: {e}")))?, + TaskState::Solved => api + .delete_solved(&key, &data.id) + .map_err(|e| Status::internal(format!("sled: {e}")))?, + TaskState::Leased => { + let mut leased = api.leased_tasks.tasks.entry(key.clone()).or_default(); + let orig = leased.len(); + leased.retain(|l| l.stored_task.id != data.id); + orig != leased.len() + } + }; + + if !removed { + return Err(Status::not_found(format!( + "Task {} not found in {:?} for {}:{}", + data.id, + data.state(), + data.namespace, + data.task, + ))); + } + + info!( + "DeleteTask: deleted {} in {:?} for {}:{}", + data.id, + data.state(), + data.namespace, + data.task + ); + Ok(tonic::Response::new(proto::Empty {})) + } + + async fn get_tasks( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + let api = self.EngineAPI.read().await; + let challenge = get_auth(&request); + + let db = api.db.clone(); + if !Events::CheckAdminAuth(&api, challenge, ("".into(), "".into()), db) { + info!("GetTask denied due to Invalid Auth"); + return Err(Status::permission_denied("Invalid authentication")); + }; + let data = request.get_ref(); + let key = ID(&data.namespace, &data.task); + let task_id_str = format!("{}:{}", data.namespace, data.task); + + let to_proto = |id: String, bytes: Vec| proto::Task { + id, + task_id: task_id_str.clone(), + task_payload: bytes, + payload: Vec::new(), + }; + + let mut tasks: Vec = match data.state() { + TaskState::Queued => { + let mut v: Vec<_> = api + .scan_queued(&key) + .map(|t| to_proto(t.id, t.bytes)) + .collect(); + v.sort_by(|a, b| a.id.cmp(&b.id)); + v + } + TaskState::Solved => { + let mut v: Vec<_> = api + .scan_solved(&key) + .map(|t| to_proto(t.id, t.bytes)) + .collect(); + v.sort_by(|a, b| a.id.cmp(&b.id)); + v + } + TaskState::Leased => { + let mut v: Vec<_> = api + .leased_tasks + .tasks + .get(&key) + .map(|leased| { + leased + .iter() + .map(|l| { + to_proto(l.stored_task.id.clone(), l.stored_task.bytes.clone()) + }) + .collect() + }) + .unwrap_or_default(); + v.sort_by(|a, b| a.id.cmp(&b.id)); + v + } + }; + + let page_size = api.cfg.config_toml.pagination_limit.min(data.page_size) as usize; + let start = (data.page as usize).saturating_mul(page_size); + let end = start.saturating_add(page_size).min(tasks.len()); + let final_vec = if start >= tasks.len() { + Vec::new() + } else { + tasks.drain(start..end).collect() + }; + + Ok(tonic::Response::new(proto::TaskPage { + namespace: data.namespace.clone(), + task: data.task.clone(), + page: data.page, + page_size: page_size as u32, + state: data.state, + tasks: final_vec, + })) + } + + async fn cgrpc( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + info!( + "CGRPC request received for handler: {}:{}", + request.get_ref().handler_mod_id, + request.get_ref().handler_id + ); + let mut api = self.EngineAPI.write().await; + let challenge = get_auth(&request); + let db = api.db.clone(); + debug!("Checking admin authentication for CGRPC request"); + let output = Events::CheckAdminAuth( + &mut api, + challenge, + ( + request.get_ref().handler_mod_id.clone(), + request.get_ref().handler_id.clone(), + ), + db, + ); + if !output { + warn!("CGRPC auth check failed - permission denied"); + return Err(tonic::Status::permission_denied("Invalid CGRPC Auth")); + }; + let out = Arc::new(std::sync::RwLock::new(Vec::new())); + debug!("Dispatching CGRPC event to handler"); + Events::CgrpcEvent( + &mut api, + ID("engine_core", "grpc"), + request.get_ref().event_payload.clone(), + out.clone(), + ); + let mut res = request.get_ref().clone(); + res.event_payload = match out.read() { + Ok(g) => g.clone(), + Err(_) => { + warn!("CGRPC response lock poisoned, returning empty payload"); + Vec::new() + } + }; + info!("CGRPC request processed successfully"); + Ok(tonic::Response::new(res)) + } + + async fn aquire_task_reg( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let uid = get_uid(&request); + let challenge = get_auth(&request); + info!("Task registry request received from user: {}", uid); + let api = self.EngineAPI.read().await; + let db = api.db.clone(); + + debug!("Validating authentication for task registry request"); + if !Events::CheckAuth(&api, uid.clone(), challenge, db) { + info!( + "Task registry request denied - invalid authentication for user: {}", + uid + ); + return Err(Status::permission_denied("Invalid authentication")); + }; + let mut tasks: Vec = Vec::new(); + for entry in api.task_registry.tasks.iter() { + let k = entry.key(); + tasks.push(format!("{}:{}", k.0, k.1)); + } + info!("Returning task registry with {} tasks", tasks.len()); + let response = proto::TaskRegistry { tasks }; + Ok(tonic::Response::new(response)) + } + + async fn aquire_task_block( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let challenge = get_auth(&request); + let task_id = request.get_ref().task_id.clone(); + let uid = get_uid(&request); + + { + let api = self.EngineAPI.read().await; + let db = api.db.clone(); + if !Events::CheckAuth(&api, uid.clone(), challenge, db) { + return Err(Status::permission_denied("Invalid authentication")); + }; + } + + let (namespace, task_name) = task_id.split_once(':').ok_or_else(|| { + Status::invalid_argument("Invalid task ID format, expected 'namespace:task'") + })?; + let key = ID(namespace, task_name); + + { + let api = self.EngineAPI.read().await; + if api.task_registry.get(&key).is_none() { + return Err(Status::invalid_argument("Task Does not Exist")); + } + if Events::ServerBeforeTaskAcquire(&api, uid.clone(), task_id.clone()) { + return Err(Status::aborted( + "Task acquire cancelled by server event handler", + )); + } + } + + let receiver = { + let api = self.EngineAPI.read().await; + api.task_queue + .tasks + .get(&key) + .map(|entry| entry.0.clone()) + .ok_or_else(|| Status::not_found("Unknown task type"))? + }; + + if receiver.is_empty() { + let lock_arc = { + let api = self.EngineAPI.read().await; + api.fill_locks.entry(key.clone()).or_default().clone() + }; + let _g = lock_arc.lock().await; + if receiver.is_empty() { + let api = self.EngineAPI.read().await; + ServerAPI::fill_queue(&api, key.clone()); + } + } + + let block = receiver + .recv() + .await + .map_err(|_| Status::unavailable("Task queue closed"))?; + + { + let api = self.EngineAPI.read().await; + let mut entry = api.leased_tasks.tasks.entry(key.clone()).or_default(); + let now = Utc::now(); + let mut ids = Vec::with_capacity(block.tasks.len()); + for task in &block.tasks { + ids.push(task.id.clone()); + entry.push(LeasedTask { + stored_task: Arc::new(task.clone()), + user_id: uid.clone(), + given_at: now, + }); + } + drop(entry); + Events::ServerTaskBlockAcquired(&api, uid.clone(), task_id.clone(), ids); + } + + let tasks = block + .tasks + .into_iter() + .map(|t| proto::Task { + id: t.id, + task_id: task_id.clone(), + task_payload: t.bytes, + payload: Vec::new(), + }) + .collect(); + + Ok(tonic::Response::new(proto::TaskBlock { tasks })) + } + + async fn publish_task_block( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let challenge = get_auth(&request); + let uid = get_uid(&request); + let api = self.EngineAPI.read().await; + + { + let db = api.db.clone(); + if !Events::CheckAuth(&api, uid.clone(), challenge, db) { + return Err(Status::permission_denied("Invalid authentication")); + }; + } + + let mut groups: HashMap> = HashMap::new(); + for t in request.into_inner().tasks { + let Some((ns, name)) = t.task_id.split_once(':') else { + info!("publish: skipping malformed task_id {}", t.task_id); + continue; + }; + groups + .entry((ns.to_string(), name.to_string())) + .or_default() + .push(t); + } + + for (key, tasks) in groups { + let task_id_str = format!("{}:{}", key.0, key.1); + let Some(reg_tsk) = api.task_registry.get(&key) else { + info!("publish: unknown task {}:{}, skipping group", key.0, key.1); + continue; + }; + + let mut published_ids: Vec = Vec::with_capacity(tasks.len()); + + for t in tasks { + let payload_for_event = Arc::new(std::sync::RwLock::new(t.task_payload.clone())); + if Events::ServerBeforeTaskPublish( + &api, + uid.clone(), + task_id_str.clone(), + t.id.clone(), + payload_for_event.clone(), + ) { + info!("publish: handler cancelled {}:{}", task_id_str, t.id); + continue; + } + + let payload = match payload_for_event.read() { + Ok(p) => p.clone(), + Err(_) => { + info!("publish: payload lock poisoned for {}", t.id); + continue; + } + }; + + if !reg_tsk.clone().verify(payload.clone()) { + info!("publish: verify failed for {}", t.id); + continue; + } + + let mut leased = api.leased_tasks.tasks.entry(key.clone()).or_default(); + let Some(idx) = leased + .iter() + .position(|l| l.stored_task.id == t.id && l.user_id == uid) + else { + info!("publish: no lease for {} held by {}", t.id, uid); + continue; + }; + leased.remove(idx); + drop(leased); + + let stored = StoredTask { + id: t.id.clone(), + bytes: payload, + }; + if let Err(e) = api.put_solved(&key, &stored) { + info!("publish: sled put_solved failed for {}: {}", t.id, e); + continue; + } + if let Err(e) = api.delete_queued(&key, &t.id) { + info!("publish: sled delete_queued failed for {}: {}", t.id, e); + } + published_ids.push(t.id); + } + + if !published_ids.is_empty() { + Events::ServerTaskBlockPublished(&api, uid.clone(), task_id_str, published_ids); + } + } + + Ok(tonic::Response::new(proto::Empty {})) + } + + async fn create_task_block( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let challenge = get_auth(&request); + let uid = get_uid(&request); + let api = self.EngineAPI.read().await; + let db = api.db.clone(); + if !Events::CheckAuth(&api, uid, challenge, db) { + info!("Create Task denied due to Invalid Auth"); + return Err(Status::permission_denied("Invalid authentication")); + }; + + let mut groups: HashMap> = HashMap::new(); + for t in request.into_inner().tasks { + let Some((ns, name)) = t.task_id.split_once(':') else { + return Err(Status::invalid_argument(format!( + "Invalid task ID format: {}", + t.task_id + ))); + }; + if ns.is_empty() || name.is_empty() { + return Err(Status::invalid_argument( + "Invalid task ID format, expected 'namespace:task'", + )); + } + groups + .entry((ns.to_string(), name.to_string())) + .or_default() + .push(t); + } + + let mut created: Vec = Vec::new(); + + for (key, tasks) in groups { + let task_id_str = format!("{}:{}", key.0, key.1); + let Some(reg_tsk) = api.task_registry.get(&key) else { + return Err(Status::invalid_argument(format!( + "Task does not exist: {}", + task_id_str + ))); + }; + + let mut block_tasks: Vec = Vec::with_capacity(tasks.len()); + let mut instance_ids: Vec = Vec::with_capacity(tasks.len()); + let mut payloads: Vec>>> = + Vec::with_capacity(tasks.len()); + + for t in tasks { + let payload_for_event = Arc::new(std::sync::RwLock::new(t.task_payload.clone())); + if Events::ServerBeforeTaskCreate( + &api, + task_id_str.clone(), + payload_for_event.clone(), + ) { + info!("create: handler cancelled task in {}", task_id_str); + continue; + } + let payload = match payload_for_event.read() { + Ok(p) => p.clone(), + Err(_) => { + info!("create: payload lock poisoned in {}", task_id_str); + continue; + } + }; + if !reg_tsk.clone().verify(payload.clone()) { + info!("create: verify failed in {}", task_id_str); + continue; + } + let stored = StoredTask { + id: druid::Druid::default().to_hex(), + bytes: payload, + }; + if let Err(e) = api.put_queued(&key, &stored) { + info!("create: sled put_queued failed for {}: {}", stored.id, e); + continue; + } + instance_ids.push(stored.id.clone()); + payloads.push(payload_for_event); + created.push(proto::Task { + id: stored.id.clone(), + task_id: task_id_str.clone(), + task_payload: stored.bytes.clone(), + payload: Vec::new(), + }); + block_tasks.push(stored); + } + + if !block_tasks.is_empty() { + if let Some(channel) = api.task_queue.tasks.get(&key) { + let _ = channel.1.try_send(StoredTaskBlock { tasks: block_tasks }); + } + Events::ServerTaskBlockCreated(&api, task_id_str, instance_ids, payloads); + } + } + + Ok(tonic::Response::new(proto::TaskBlock { tasks: created })) + } +} diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index b556c59..11b04f4 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -85,6 +85,16 @@ impl ServerAPI { fill_locks: DashMap::new(), } } + /// Ensure the task_queue + leased_tasks entries exist for this Identifier. + /// Idempotent — calling repeatedly is a no-op once registered. + pub fn ensure_task_channel(&self, id: Identifier) { + self.task_queue.tasks.entry(id.clone()).or_insert_with(|| { + let (s, r) = async_channel::unbounded(); + (r, s) + }); + self.leased_tasks.tasks.entry(id).or_default(); + } + pub fn init(api: &mut Self) { Self::setup_logger(); api.cfg = Config::new(); @@ -101,6 +111,46 @@ impl ServerAPI { Self::init_events(api); } + /// Client-side ServerAPI with a temp sled (client doesn't use db/task_queue + /// at all; the temp dir is just to satisfy the struct field). + pub fn default_client() -> Self { + use std::sync::atomic::{AtomicUsize, Ordering}; + static DB_COUNTER: AtomicUsize = AtomicUsize::new(0); + let db_id = DB_COUNTER.fetch_add(1, Ordering::Relaxed); + let db_path = std::env::temp_dir().join(format!( + "enginelib-client-db-{}-{}", + std::process::id(), + db_id + )); + Self { + cfg: Config::new(), + task_queue: TaskQueue::default(), + leased_tasks: LeasedTaskQueue::default(), + task_registry: EngineTaskRegistry::default(), + event_bus: EventBus { + event_handler_registry: EngineEventHandlerRegistry { + event_handlers: HashMap::new(), + }, + }, + db: sled::Config::new() + .path(db_path) + .temporary(true) + .flush_every_ms(None) + .open() + .unwrap(), + lib_manager: LibraryManager::default(), + fill_locks: DashMap::new(), + } + } + + /// Client init: logger + inventory event handlers. Skips load_modules + /// because client mods are loaded via a different path and module + /// validation happens against server metadata. + pub fn init_client(api: &mut Self) { + Self::setup_logger(); + Self::init_events(api); + } + fn init_events(api: &mut Self) { crate::event::register_inventory_handlers(api); } diff --git a/enginelib/src/events/before_task_acquire_event.rs b/enginelib/src/events/before_task_acquire_event.rs deleted file mode 100644 index 9b85683..0000000 --- a/enginelib/src/events/before_task_acquire_event.rs +++ /dev/null @@ -1,27 +0,0 @@ -use macros::Event; - -use crate::{Identifier, api::ServerAPI}; - -#[derive(Clone, Debug, Event)] -#[event(namespace = "client", name = "before_task_acquire", cancellable)] -pub struct BeforeTaskAcquireEvent { - pub cancelled: bool, - pub id: Identifier, - pub task_id: String, -} - -impl BeforeTaskAcquireEvent { - pub fn fire(api: &ServerAPI, task_id: String) -> Self { - let mut event = BeforeTaskAcquireEvent { - cancelled: false, - id: ("client".to_string(), "before_task_acquire".to_string()), - task_id, - }; - api.event_bus.fire(&mut event); - event - } - - pub fn check(api: &ServerAPI, task_id: String) -> bool { - Self::fire(api, task_id).cancelled - } -} diff --git a/enginelib/src/events/before_task_block_acquire_event.rs b/enginelib/src/events/before_task_block_acquire_event.rs new file mode 100644 index 0000000..13a79f1 --- /dev/null +++ b/enginelib/src/events/before_task_block_acquire_event.rs @@ -0,0 +1,30 @@ +use macros::Event; + +use crate::{Identifier, api::ServerAPI}; + +#[derive(Clone, Debug, Event)] +#[event(namespace = "client", name = "before_task_block_acquire", cancellable)] +pub struct BeforeTaskBlockAcquireEvent { + pub cancelled: bool, + pub id: Identifier, + pub task_ids: Vec, +} + +impl BeforeTaskBlockAcquireEvent { + pub fn fire(api: &ServerAPI, task_ids: Vec) -> Self { + let mut event = BeforeTaskBlockAcquireEvent { + cancelled: false, + id: ( + "client".to_string(), + "before_task_block_acquire".to_string(), + ), + task_ids, + }; + api.event_bus.fire(&mut event); + event + } + + pub fn check(api: &ServerAPI, task_ids: Vec) -> bool { + Self::fire(api, task_ids).cancelled + } +} diff --git a/enginelib/src/events/before_task_block_execute_event.rs b/enginelib/src/events/before_task_block_execute_event.rs new file mode 100644 index 0000000..5da0ccc --- /dev/null +++ b/enginelib/src/events/before_task_block_execute_event.rs @@ -0,0 +1,46 @@ +use std::sync::{Arc, RwLock}; + +use macros::Event; + +use crate::{Identifier, api::ServerAPI}; + +#[derive(Clone, Debug, Event)] +#[event(namespace = "client", name = "before_task_block_execute", cancellable)] +pub struct BeforeTaskBlockExecuteEvent { + pub cancelled: bool, + pub id: Identifier, + pub task_id: String, + pub instance_ids: Vec, + pub payloads: Vec>>>, +} + +impl BeforeTaskBlockExecuteEvent { + pub fn fire( + api: &ServerAPI, + task_id: String, + instance_ids: Vec, + payloads: Vec>>>, + ) -> Self { + let mut event = BeforeTaskBlockExecuteEvent { + cancelled: false, + id: ( + "client".to_string(), + "before_task_block_execute".to_string(), + ), + task_id, + instance_ids, + payloads, + }; + api.event_bus.fire(&mut event); + event + } + + pub fn check( + api: &ServerAPI, + task_id: String, + instance_ids: Vec, + payloads: Vec>>>, + ) -> bool { + Self::fire(api, task_id, instance_ids, payloads).cancelled + } +} diff --git a/enginelib/src/events/before_task_block_publish_event.rs b/enginelib/src/events/before_task_block_publish_event.rs new file mode 100644 index 0000000..c97cb64 --- /dev/null +++ b/enginelib/src/events/before_task_block_publish_event.rs @@ -0,0 +1,46 @@ +use std::sync::{Arc, RwLock}; + +use macros::Event; + +use crate::{Identifier, api::ServerAPI}; + +#[derive(Clone, Debug, Event)] +#[event(namespace = "client", name = "before_task_block_publish", cancellable)] +pub struct BeforeTaskBlockPublishEvent { + pub cancelled: bool, + pub id: Identifier, + pub task_id: String, + pub instance_ids: Vec, + pub payloads: Vec>>>, +} + +impl BeforeTaskBlockPublishEvent { + pub fn fire( + api: &ServerAPI, + task_id: String, + instance_ids: Vec, + payloads: Vec>>>, + ) -> Self { + let mut event = BeforeTaskBlockPublishEvent { + cancelled: false, + id: ( + "client".to_string(), + "before_task_block_publish".to_string(), + ), + task_id, + instance_ids, + payloads, + }; + api.event_bus.fire(&mut event); + event + } + + pub fn check( + api: &ServerAPI, + task_id: String, + instance_ids: Vec, + payloads: Vec>>>, + ) -> bool { + Self::fire(api, task_id, instance_ids, payloads).cancelled + } +} diff --git a/enginelib/src/events/before_task_execute_event.rs b/enginelib/src/events/before_task_execute_event.rs deleted file mode 100644 index 07e4d63..0000000 --- a/enginelib/src/events/before_task_execute_event.rs +++ /dev/null @@ -1,43 +0,0 @@ -use std::sync::{Arc, RwLock}; - -use macros::Event; - -use crate::{Identifier, api::ServerAPI}; - -#[derive(Clone, Debug, Event)] -#[event(namespace = "client", name = "before_task_execute", cancellable)] -pub struct BeforeTaskExecuteEvent { - pub cancelled: bool, - pub id: Identifier, - pub task_id: String, - pub instance_id: String, - pub payload: Arc>>, -} - -impl BeforeTaskExecuteEvent { - pub fn fire( - api: &ServerAPI, - task_id: String, - instance_id: String, - payload: Arc>>, - ) -> Self { - let mut event = BeforeTaskExecuteEvent { - cancelled: false, - id: ("client".to_string(), "before_task_execute".to_string()), - task_id, - instance_id, - payload, - }; - api.event_bus.fire(&mut event); - event - } - - pub fn check( - api: &ServerAPI, - task_id: String, - instance_id: String, - payload: Arc>>, - ) -> bool { - Self::fire(api, task_id, instance_id, payload).cancelled - } -} diff --git a/enginelib/src/events/before_task_publish_event.rs b/enginelib/src/events/before_task_publish_event.rs deleted file mode 100644 index f3c31dd..0000000 --- a/enginelib/src/events/before_task_publish_event.rs +++ /dev/null @@ -1,43 +0,0 @@ -use std::sync::{Arc, RwLock}; - -use macros::Event; - -use crate::{Identifier, api::ServerAPI}; - -#[derive(Clone, Debug, Event)] -#[event(namespace = "client", name = "before_task_publish", cancellable)] -pub struct BeforeTaskPublishEvent { - pub cancelled: bool, - pub id: Identifier, - pub task_id: String, - pub instance_id: String, - pub payload: Arc>>, -} - -impl BeforeTaskPublishEvent { - pub fn fire( - api: &ServerAPI, - task_id: String, - instance_id: String, - payload: Arc>>, - ) -> Self { - let mut event = BeforeTaskPublishEvent { - cancelled: false, - id: ("client".to_string(), "before_task_publish".to_string()), - task_id, - instance_id, - payload, - }; - api.event_bus.fire(&mut event); - event - } - - pub fn check( - api: &ServerAPI, - task_id: String, - instance_id: String, - payload: Arc>>, - ) -> bool { - Self::fire(api, task_id, instance_id, payload).cancelled - } -} diff --git a/enginelib/src/events/mod.rs b/enginelib/src/events/mod.rs index 7c687d6..6b39213 100644 --- a/enginelib/src/events/mod.rs +++ b/enginelib/src/events/mod.rs @@ -1,8 +1,8 @@ pub mod admin_auth_event; pub mod auth_event; -pub mod before_task_acquire_event; -pub mod before_task_execute_event; -pub mod before_task_publish_event; +pub mod before_task_block_acquire_event; +pub mod before_task_block_execute_event; +pub mod before_task_block_publish_event; pub mod cgrpc_event; pub mod client_auth_prepare_event; pub mod client_start_event; @@ -14,7 +14,7 @@ pub mod server_task_block_acquired_event; pub mod server_task_block_created_event; pub mod server_task_block_published_event; pub mod start_event; -pub mod task_acquired_event; +pub mod task_block_acquired_event; use std::collections::HashMap; use std::sync::{Arc, RwLock}; @@ -36,12 +36,7 @@ impl Events { auth_event::AuthEvent::check(api, uid, challenge, db) } - pub fn CheckAdminAuth( - api: &ServerAPI, - payload: String, - target: Identifier, - db: Db, - ) -> bool { + pub fn CheckAdminAuth(api: &ServerAPI, payload: String, target: Identifier, db: Db) -> bool { admin_auth_event::AdminAuthEvent::check(api, payload, target, db) } @@ -66,35 +61,50 @@ impl Events { client_auth_prepare_event::ClientAuthPrepareEvent::fire(api, headers) } - pub fn BeforeTaskAcquire(api: &ServerAPI, task_id: String) -> bool { - before_task_acquire_event::BeforeTaskAcquireEvent::check(api, task_id) + pub fn BeforeTaskBlockAcquire(api: &ServerAPI, task_ids: Vec) -> bool { + before_task_block_acquire_event::BeforeTaskBlockAcquireEvent::check(api, task_ids) } - pub fn TaskAcquired( + pub fn TaskBlockAcquired( api: &ServerAPI, task_id: String, - instance_id: String, - payload: Arc>>, + instance_ids: Vec, + payloads: Vec>>>, ) { - task_acquired_event::TaskAcquiredEvent::fire(api, task_id, instance_id, payload) + task_block_acquired_event::TaskBlockAcquiredEvent::fire( + api, + task_id, + instance_ids, + payloads, + ) } - pub fn BeforeTaskExecute( + pub fn BeforeTaskBlockExecute( api: &ServerAPI, task_id: String, - instance_id: String, - payload: Arc>>, + instance_ids: Vec, + payloads: Vec>>>, ) -> bool { - before_task_execute_event::BeforeTaskExecuteEvent::check(api, task_id, instance_id, payload) + before_task_block_execute_event::BeforeTaskBlockExecuteEvent::check( + api, + task_id, + instance_ids, + payloads, + ) } - pub fn BeforeTaskPublish( + pub fn BeforeTaskBlockPublish( api: &ServerAPI, task_id: String, - instance_id: String, - payload: Arc>>, + instance_ids: Vec, + payloads: Vec>>>, ) -> bool { - before_task_publish_event::BeforeTaskPublishEvent::check(api, task_id, instance_id, payload) + before_task_block_publish_event::BeforeTaskBlockPublishEvent::check( + api, + task_id, + instance_ids, + payloads, + ) } pub fn ServerStart(api: &ServerAPI) { diff --git a/enginelib/src/events/task_acquired_event.rs b/enginelib/src/events/task_acquired_event.rs deleted file mode 100644 index 7113cac..0000000 --- a/enginelib/src/events/task_acquired_event.rs +++ /dev/null @@ -1,33 +0,0 @@ -use std::sync::{Arc, RwLock}; - -use macros::Event; - -use crate::{Identifier, api::ServerAPI}; - -#[derive(Clone, Debug, Event)] -#[event(namespace = "client", name = "task_acquired")] -pub struct TaskAcquiredEvent { - pub cancelled: bool, - pub id: Identifier, - pub task_id: String, - pub instance_id: String, - pub payload: Arc>>, -} - -impl TaskAcquiredEvent { - pub fn fire( - api: &ServerAPI, - task_id: String, - instance_id: String, - payload: Arc>>, - ) { - let mut event = TaskAcquiredEvent { - cancelled: false, - id: ("client".to_string(), "task_acquired".to_string()), - task_id, - instance_id, - payload, - }; - api.event_bus.fire(&mut event); - } -} diff --git a/enginelib/src/events/task_block_acquired_event.rs b/enginelib/src/events/task_block_acquired_event.rs new file mode 100644 index 0000000..4b3be50 --- /dev/null +++ b/enginelib/src/events/task_block_acquired_event.rs @@ -0,0 +1,33 @@ +use std::sync::{Arc, RwLock}; + +use macros::Event; + +use crate::{Identifier, api::ServerAPI}; + +#[derive(Clone, Debug, Event)] +#[event(namespace = "client", name = "task_block_acquired")] +pub struct TaskBlockAcquiredEvent { + pub cancelled: bool, + pub id: Identifier, + pub task_id: String, + pub instance_ids: Vec, + pub payloads: Vec>>>, +} + +impl TaskBlockAcquiredEvent { + pub fn fire( + api: &ServerAPI, + task_id: String, + instance_ids: Vec, + payloads: Vec>>>, + ) { + let mut event = TaskBlockAcquiredEvent { + cancelled: false, + id: ("client".to_string(), "task_block_acquired".to_string()), + task_id, + instance_ids, + payloads, + }; + api.event_bus.fire(&mut event); + } +} diff --git a/enginelib/src/prelude.rs b/enginelib/src/prelude.rs index c1af52d..899d880 100644 --- a/enginelib/src/prelude.rs +++ b/enginelib/src/prelude.rs @@ -12,9 +12,9 @@ pub use crate::event::{ }; pub use crate::events::admin_auth_event::AdminAuthEvent; pub use crate::events::auth_event::AuthEvent; -pub use crate::events::before_task_acquire_event::BeforeTaskAcquireEvent; -pub use crate::events::before_task_execute_event::BeforeTaskExecuteEvent; -pub use crate::events::before_task_publish_event::BeforeTaskPublishEvent; +pub use crate::events::before_task_block_acquire_event::BeforeTaskBlockAcquireEvent; +pub use crate::events::before_task_block_execute_event::BeforeTaskBlockExecuteEvent; +pub use crate::events::before_task_block_publish_event::BeforeTaskBlockPublishEvent; pub use crate::events::cgrpc_event::CgrpcEvent; pub use crate::events::client_auth_prepare_event::ClientAuthPrepareEvent; pub use crate::events::client_start_event::ClientStartEvent; @@ -26,7 +26,7 @@ pub use crate::events::server_task_block_acquired_event::ServerTaskBlockAcquired pub use crate::events::server_task_block_created_event::ServerTaskBlockCreatedEvent; pub use crate::events::server_task_block_published_event::ServerTaskBlockPublishedEvent; pub use crate::events::start_event::StartEvent; -pub use crate::events::task_acquired_event::TaskAcquiredEvent; +pub use crate::events::task_block_acquired_event::TaskBlockAcquiredEvent; pub use crate::events::{Events, ID}; pub use crate::plugin::{LibraryDependency, LibraryMetadata}; pub use crate::task::{Runner, Task, Verifiable}; From fdca4ac0ebef7268c93581e37f679500e3cc17c3 Mon Sep 17 00:00:00 2001 From: IGN-Styly Date: Mon, 1 Jun 2026 22:19:19 +0100 Subject: [PATCH 17/28] optimize --- Cargo.lock | 34 ++ engine/Cargo.toml | 3 + engine/proto/engine.proto | 2 + engine/src/bin/bench.rs | 306 ++++++++++++++---- engine/src/bin/client.rs | 101 ++++-- engine/src/lib.rs | 456 +++++++++++++++++---------- engine/tests/task_streaming_tests.rs | 180 +++++++++++ enginelib/src/api.rs | 121 +++++-- 8 files changed, 929 insertions(+), 274 deletions(-) create mode 100644 engine/tests/task_streaming_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 1f3eb0f..240d8d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -676,8 +676,11 @@ dependencies = [ "druid", "enginelib", "prost", + "rayon", "serde", + "sha2", "tokio", + "tokio-stream", "toml", "tonic", "tonic-build", @@ -2456,6 +2459,26 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2649,6 +2672,17 @@ dependencies = [ "sha1", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" diff --git a/engine/Cargo.toml b/engine/Cargo.toml index 792bac7..9bc39c9 100644 --- a/engine/Cargo.toml +++ b/engine/Cargo.toml @@ -17,9 +17,12 @@ druid = { git = "https://github.com/GrandEngineering/druid.git" } enginelib = { path = "../enginelib" } # libloading = "0.8.6" prost = "0.14" +rayon = "1.10" serde = { workspace = true } +sha2 = "0.10" # serde = "1.0.219" tokio = { version = "1.50.0", features = ["rt-multi-thread", "macros"] } +tokio-stream = "0.1.17" toml = { workspace = true } # toml = "0.8.19" tonic = "0.14" diff --git a/engine/proto/engine.proto b/engine/proto/engine.proto index 6af5c55..bad8a33 100644 --- a/engine/proto/engine.proto +++ b/engine/proto/engine.proto @@ -2,8 +2,10 @@ syntax = "proto3"; package engine; service Engine { rpc AquireTaskBlock(TaskBlockRequest) returns (TaskBlock); + rpc AquireTaskStream(TaskBlockRequest) returns (stream Task); rpc AquireTaskReg(empty) returns (TaskRegistry); rpc PublishTaskBlock(TaskBlock) returns (empty); + rpc PublishTaskStream(stream Task) returns (empty); rpc cgrpc(cgrpcmsg) returns (cgrpcmsg); rpc CreateTaskBlock(TaskBlock) returns (TaskBlock); rpc DeleteTaskBlock(TaskSelector) returns (empty); diff --git a/engine/src/bin/bench.rs b/engine/src/bin/bench.rs index 8ca8696..29a8ff0 100644 --- a/engine/src/bin/bench.rs +++ b/engine/src/bin/bench.rs @@ -1,9 +1,9 @@ -// End-to-end TPS benchmark for the block-based gRPC pipeline. +// End-to-end TPS benchmark for the PoW task gRPC pipeline. // // Spins the engine server in-process on a localhost ephemeral port, registers a -// no-op BenchTask, then runs two phases over real loopback gRPC: +// local copy of the engine_mod:pow task, then runs two phases over real loopback gRPC: // 1. Enqueue N tasks via CreateTaskBlock in chunks -// 2. Drain via W concurrent workers (acquire_task_block + publish_task_block) +// 2. Drain via W concurrent workers (aquire_task_stream + publish_task_stream) // Prints per-phase elapsed/TPS and a linear extrapolation to 1B tasks. use clap::Parser; @@ -14,8 +14,13 @@ use enginelib::{ task::{Task, Verifiable}, }; use proto::engine_client; +use rayon::prelude::*; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use std::{ + fs, net::{Ipv4Addr, SocketAddr, SocketAddrV4}, + process::Command, sync::{ Arc, atomic::{AtomicUsize, Ordering}, @@ -29,8 +34,8 @@ use tokio::{ }; use tonic::{Request, transport::Server}; -const BENCH_NS: &str = "bench"; -const BENCH_TASK: &str = "noop"; +const BENCH_NS: &str = "engine_mod"; +const BENCH_TASK: &str = "pow"; fn parse_nonzero_usize(value: &str) -> Result { let parsed = value @@ -74,15 +79,24 @@ struct Args { /// Worker count for the process phase. #[arg(long, default_value_t = num_cpus_fallback(), value_parser = parse_nonzero_usize)] workers: usize, + /// Rayon CPU worker count used to solve tasks inside acquired blocks. + #[arg(long, default_value_t = num_cpus_fallback(), value_parser = parse_nonzero_usize)] + compute_workers: usize, /// Block size for enqueue (CreateTaskBlock chunks). #[arg(long, default_value_t = 1024, value_parser = parse_nonzero_u32)] block_size: u32, - /// Payload size in bytes per task (zero-filled). - #[arg(long, default_value_t = 16)] - payload_bytes: usize, + /// PoW leading zero nibbles required by each task. + #[arg(long, default_value_t = 1)] + pow_zeros: u8, + /// First u32 seed to use when generating PoW task payloads. + #[arg(long, default_value_t = 0)] + seed_start: u32, /// Per-acquire timeout so workers can recover if the queue unexpectedly stalls. #[arg(long, default_value_t = 1000, value_parser = parse_nonzero_u64)] acquire_timeout_ms: u64, + /// Legacy no-op benchmark argument retained for old command lines. + #[arg(long, hide = true)] + _payload_bytes: Option, /// Cargo passes this to harness-free benchmark targets. #[arg(long = "bench", hide = true, action = clap::ArgAction::SetTrue)] _cargo_bench: bool, @@ -94,35 +108,77 @@ fn num_cpus_fallback() -> usize { .unwrap_or(4) } -#[derive(Debug, Clone, Default)] -struct BenchTask; +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +struct PowTask { + seed: u32, + zeros: u8, + nonce: u64, +} + +impl PowTask { + #[inline] + fn has_leading_zero_nibbles(digest: &[u8; 32], zeros: u8) -> bool { + let zeros = usize::from(zeros).min(64); + let full_zero_bytes = zeros / 2; + + if digest[..full_zero_bytes].iter().any(|&b| b != 0) { + return false; + } + + if zeros % 2 == 1 { + (digest[full_zero_bytes] & 0xF0) == 0 + } else { + true + } + } +} -impl Verifiable for BenchTask { - fn verify(&self, _b: Vec) -> bool { - true +impl Verifiable for PowTask { + fn verify(&self, b: Vec) -> bool { + enginelib::api::postcard::from_bytes::(&b).is_ok() } } -impl Task for BenchTask { +impl Task for PowTask { fn get_id(&self) -> Identifier { (BENCH_NS.to_string(), BENCH_TASK.to_string()) } + fn clone_box(&self) -> Box { Box::new(self.clone()) } - fn run_hip(&mut self) {} - fn run_cpu(&mut self) {} + + fn run_cpu(&mut self) { + let zeros = self.zeros.min(64); + let seed_bytes = self.seed.to_le_bytes(); + + for n in 0..=u64::MAX { + let mut hasher = Sha256::new(); + hasher.update(n.to_le_bytes()); + hasher.update(seed_bytes); + let digest: [u8; 32] = hasher.finalize().into(); + + if Self::has_leading_zero_nibbles(&digest, zeros) { + self.nonce = n; + return; + } + } + } + fn to_bytes(&self) -> Vec { - Vec::new() + enginelib::api::to_allocvec(self).unwrap() } - fn from_bytes(&self, _bytes: &[u8]) -> Box { - Box::new(self.clone()) + + fn from_bytes(&self, bytes: &[u8]) -> Box { + Box::new(enginelib::api::from_bytes::(bytes).unwrap()) } - fn from_toml(&self, _d: String) -> Box { - Box::new(self.clone()) + + fn from_toml(&self, d: String) -> Box { + Box::new(toml::from_str::(&d).unwrap()) } + fn to_toml(&self) -> String { - String::new() + toml::to_string(self).unwrap() } } @@ -132,10 +188,12 @@ async fn pick_port() -> u16 { listener.local_addr().unwrap().port() } -async fn build_server_api() -> Arc> { +async fn build_server_api(block_size: u32) -> Arc> { let mut api = ServerAPI::test_default(); + api.cfg.config_toml.task_block_size = block_size; let id: Identifier = (BENCH_NS.to_string(), BENCH_TASK.to_string()); - api.task_registry.register(Arc::new(BenchTask), id.clone()); + api.task_registry + .register(Arc::new(PowTask::default()), id.clone()); api.ensure_task_channel(id); // Wire up inventory event handlers — the default core::auth_event handler // approves auth, which the bench client relies on (it sends no creds). @@ -143,8 +201,8 @@ async fn build_server_api() -> Arc> { Arc::new(RwLock::new(api)) } -async fn spawn_server(port: u16) -> Arc> { - let api = build_server_api().await; +async fn spawn_server(port: u16, block_size: u32) -> Arc> { + let api = build_server_api(block_size).await; let engine = EngineService::new(api.clone()); let addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), port)); tokio::spawn(async move { @@ -196,11 +254,49 @@ fn fmt_tps(tps: f64) -> String { } } +fn clock_ticks_per_second() -> f64 { + Command::new("getconf") + .arg("CLK_TCK") + .output() + .ok() + .and_then(|output| String::from_utf8(output.stdout).ok()) + .and_then(|text| text.trim().parse::().ok()) + .filter(|ticks| *ticks > 0.0) + .unwrap_or(100.0) +} + +fn process_cpu_time() -> Option { + let stat = fs::read_to_string("/proc/self/stat").ok()?; + let fields = stat.rsplit_once(") ")?.1; + let values: Vec<&str> = fields.split_whitespace().collect(); + let utime = values.get(11)?.parse::().ok()?; + let stime = values.get(12)?.parse::().ok()?; + let secs = (utime + stime) as f64 / clock_ticks_per_second(); + Some(Duration::from_secs_f64(secs)) +} + +fn cpu_delta_since(start: Option) -> Option { + Some(process_cpu_time()?.saturating_sub(start?)) +} + +fn fmt_cpu(cpu: Option, wall: Duration) -> String { + let Some(cpu) = cpu else { + return "cpu n/a".to_string(); + }; + let pct = if wall.is_zero() { + 0.0 + } else { + cpu.as_secs_f64() / wall.as_secs_f64() * 100.0 + }; + format!("cpu {} ({pct:.0}%)", fmt_dur(cpu)) +} + async fn enqueue_phase( client: &mut engine_client::EngineClient, total: usize, block_size: u32, - payload_bytes: usize, + zeros: u8, + seed_start: u32, ) -> Duration { let task_id = format!("{}:{}", BENCH_NS, BENCH_TASK); let start = Instant::now(); @@ -208,11 +304,18 @@ async fn enqueue_phase( while sent < total { let chunk = (block_size as usize).min(total - sent); let tasks: Vec = (0..chunk) - .map(|_| proto::Task { - id: String::new(), - task_id: task_id.clone(), - task_payload: vec![0u8; payload_bytes], - payload: Vec::new(), + .map(|i| { + let task = PowTask { + seed: seed_start.wrapping_add((sent + i) as u32), + zeros, + nonce: 0, + }; + proto::Task { + id: String::new(), + task_id: task_id.clone(), + task_payload: task.to_bytes(), + payload: Vec::new(), + } }) .collect(); client @@ -224,22 +327,46 @@ async fn enqueue_phase( start.elapsed() } +#[derive(Debug)] +struct ProcessReport { + elapsed: Duration, + blocks_per_worker: Vec, +} + +fn solve_pow_proto_task(task: proto::Task) -> Result { + let mut pow = enginelib::api::from_bytes::(&task.task_payload) + .map_err(|err| format!("invalid PoW payload for {}: {err}", task.id))?; + pow.run_cpu(); + Ok(proto::Task { + id: task.id, + task_id: task.task_id, + task_payload: pow.to_bytes(), + payload: Vec::new(), + }) +} + async fn process_phase( port: u16, total: usize, workers: usize, block_size: u32, acquire_timeout: Duration, -) -> Result { +) -> Result { let task_id = format!("{}:{}", BENCH_NS, BENCH_TASK); let counter = Arc::new(AtomicUsize::new(0)); + let blocks_per_worker = Arc::new( + (0..workers) + .map(|_| AtomicUsize::new(0)) + .collect::>(), + ); let (done_tx, done_rx) = watch::channel(false); let start = Instant::now(); let mut handles = JoinSet::new(); - for _ in 0..workers { + for worker_id in 0..workers { let task_id = task_id.clone(); let counter = counter.clone(); + let blocks_per_worker = blocks_per_worker.clone(); let done_tx = done_tx.clone(); let mut done_rx = done_rx.clone(); handles.spawn(async move { @@ -251,7 +378,7 @@ async fn process_phase( // The server-side acquire waits on a queue receive. The timeout // catches unexpected stalls, while the finished watch signal // keeps normal completion from paying the full timeout tail. - let acquire = client.aquire_task_block(Request::new(proto::TaskBlockRequest { + let acquire = client.aquire_task_stream(Request::new(proto::TaskBlockRequest { task_id: task_id.clone(), block_size, })); @@ -264,7 +391,7 @@ async fn process_phase( } resp = tokio::time::timeout(acquire_timeout, acquire) => resp, }; - let block = match resp { + let mut task_stream = match resp { Ok(Ok(r)) => r.into_inner(), Ok(Err(status)) => { if counter.load(Ordering::Relaxed) >= total { @@ -285,15 +412,33 @@ async fn process_phase( continue; } }; - if block.tasks.is_empty() { + + let mut acquired = Vec::with_capacity(block_size as usize); + loop { + match task_stream.message().await { + Ok(Some(task)) => acquired.push(task), + Ok(None) => break, + Err(status) => { + return Err(format!("acquire stream failed for {task_id}: {status}")); + } + } + } + + if acquired.is_empty() { continue; } - let n = block.tasks.len(); - // Echo the block right back as the publish. BenchTask::run_hip is - // a no-op so we're measuring pure RPC + storage overhead. - let solved: Vec = block.tasks; + let n = acquired.len(); + blocks_per_worker[worker_id].fetch_add(1, Ordering::Relaxed); + let solved: Vec = tokio::task::spawn_blocking(move || { + acquired + .into_par_iter() + .map(solve_pow_proto_task) + .collect::>() + }) + .await + .map_err(|err| format!("PoW worker task failed: {err}"))??; match client - .publish_task_block(Request::new(proto::TaskBlock { tasks: solved })) + .publish_task_stream(Request::new(tokio_stream::iter(solved))) .await { Ok(_) => { @@ -325,34 +470,61 @@ async fn process_phase( } } } - Ok(start.elapsed()) + Ok(ProcessReport { + elapsed: start.elapsed(), + blocks_per_worker: blocks_per_worker + .iter() + .map(|count| count.load(Ordering::Relaxed)) + .collect(), + }) } #[tokio::main(flavor = "multi_thread")] async fn main() { let args = Args::parse(); + let _ = rayon::ThreadPoolBuilder::new() + .num_threads(args.compute_workers) + .thread_name(|idx| format!("engine-bench-cpu-{idx}")) + .build_global(); println!( - "engine bench: tasks={} workers={} block_size={} payload_bytes={}", - args.tasks, args.workers, args.block_size, args.payload_bytes + "engine bench: task={}:{} tasks={} workers={} compute_workers={} block_size={} pow_zeros={} seed_start={}", + BENCH_NS, + BENCH_TASK, + args.tasks, + args.workers, + args.compute_workers, + args.block_size, + args.pow_zeros, + args.seed_start ); let port = pick_port().await; - let _server_api = spawn_server(port).await; + let _server_api = spawn_server(port, args.block_size).await; let mut client = build_client(port).await; println!("Phase 1: enqueue"); - let enq = enqueue_phase(&mut client, args.tasks, args.block_size, args.payload_bytes).await; + let enq_cpu_start = process_cpu_time(); + let enq = enqueue_phase( + &mut client, + args.tasks, + args.block_size, + args.pow_zeros, + args.seed_start, + ) + .await; let enq_tps = args.tasks as f64 / enq.as_secs_f64(); println!( - " {} tasks in {} ({})", + " {} tasks in {} ({}, {})", args.tasks, fmt_dur(enq), - fmt_tps(enq_tps) + fmt_tps(enq_tps), + fmt_cpu(cpu_delta_since(enq_cpu_start), enq) ); println!("Phase 2: process"); - let proc = process_phase( + let proc_cpu_start = process_cpu_time(); + let proc_report = process_phase( port, args.tasks, args.workers, @@ -361,12 +533,37 @@ async fn main() { ) .await .expect("process phase failed"); + let proc = proc_report.elapsed; let proc_tps = args.tasks as f64 / proc.as_secs_f64(); println!( - " {} tasks in {} ({})", + " {} tasks in {} ({}, {})", args.tasks, fmt_dur(proc), - fmt_tps(proc_tps) + fmt_tps(proc_tps), + fmt_cpu(cpu_delta_since(proc_cpu_start), proc) + ); + let active_workers = proc_report + .blocks_per_worker + .iter() + .filter(|blocks| **blocks > 0) + .count(); + let total_blocks: usize = proc_report.blocks_per_worker.iter().sum(); + let min_nonzero_blocks = proc_report + .blocks_per_worker + .iter() + .copied() + .filter(|blocks| *blocks > 0) + .min() + .unwrap_or(0); + let max_blocks = proc_report + .blocks_per_worker + .iter() + .copied() + .max() + .unwrap_or(0); + println!( + " worker blocks: total={} active_workers={}/{} min_nonzero={} max={}", + total_blocks, active_workers, args.workers, min_nonzero_blocks, max_blocks ); let total = enq + proc; @@ -397,7 +594,10 @@ async fn main() { println!("\nCaveats: linear extrapolation assumes:"); println!(" - sled scales linearly with prefix size (it doesn't past tens of GB)"); - println!(" - leased_tasks DashMap memory is bounded (here it churns; at 1B it spikes)"); + println!( + " - zeros={} PoW difficulty stays representative", + args.pow_zeros + ); println!(" - task_queue_size soft cap doesn't throttle producers"); println!(" - loopback gRPC ≈ real network (it's typically 2-5x faster than LAN)"); } diff --git a/engine/src/bin/client.rs b/engine/src/bin/client.rs index 85d5c02..0e04de0 100644 --- a/engine/src/bin/client.rs +++ b/engine/src/bin/client.rs @@ -2,6 +2,7 @@ use enginelib::{ Registry, api::ServerAPI, event::info, events::Events, plugin::LibraryInstance, prelude::debug, }; use proto::engine_client; +use rayon::prelude::*; use std::{collections::HashMap, error::Error, sync::Arc}; use tonic::{ Request, @@ -63,17 +64,14 @@ async fn worker_loop( } let resp = client - .aquire_task_block(Request::new(proto::TaskBlockRequest { + .aquire_task_stream(Request::new(proto::TaskBlockRequest { task_id: task_id.clone(), block_size: 0, })) .await; - let block = match resp { - Ok(r) => { - got_any = true; - r.into_inner() - } + let mut task_stream = match resp { + Ok(r) => r.into_inner(), Err(status) if status.code() == tonic::Code::NotFound => continue, Err(status) if status.code() == tonic::Code::PermissionDenied => { debug!( @@ -92,9 +90,27 @@ async fn worker_loop( } }; + let mut tasks = Vec::new(); + loop { + match task_stream.message().await { + Ok(Some(task)) => tasks.push(task), + Ok(None) => break, + Err(status) => { + debug!( + "worker {}: acquire stream failed for {}: {:?}", + worker_id, task_id, status + ); + tasks.clear(); + break; + } + } + } + + let block = proto::TaskBlock { tasks }; if block.tasks.is_empty() { continue; } + got_any = true; let identifier = match task_id.split_once(':') { Some(v) => (v.0.to_string(), v.1.to_string()), @@ -128,25 +144,46 @@ async fn worker_loop( continue; } - let mut solved: Vec = Vec::with_capacity(block.tasks.len()); - let mut publish_payload_locks: Vec>>> = - Vec::with_capacity(block.tasks.len()); - for (t, payload_lock) in block.tasks.iter().zip(acquired_payloads.iter()) { - let payload = match payload_lock.read() { - Ok(p) => p.clone(), - Err(_) => continue, - }; - let mut tsk = task_def.clone().from_bytes(&payload); - tsk.run_hip(); - let out_bytes = tsk.to_bytes(); - publish_payload_locks.push(Arc::new(std::sync::RwLock::new(out_bytes.clone()))); - solved.push(proto::Task { - id: t.id.clone(), - task_id: task_id.clone(), - task_payload: out_bytes, - payload: Vec::new(), - }); - } + let task_id_for_execute = task_id.clone(); + let tasks_for_execute = block.tasks; + let payloads_for_execute = acquired_payloads.clone(); + let task_def_for_execute = task_def.clone(); + let execute_result = tokio::task::spawn_blocking(move || { + tasks_for_execute + .into_par_iter() + .zip(payloads_for_execute.into_par_iter()) + .filter_map(|(t, payload_lock)| { + let payload = match payload_lock.read() { + Ok(p) => p.clone(), + Err(_) => return None, + }; + let mut tsk = task_def_for_execute.clone().from_bytes(&payload); + tsk.run_hip(); + let out_bytes = tsk.to_bytes(); + let publish_payload_lock = + Arc::new(std::sync::RwLock::new(out_bytes.clone())); + let solved = proto::Task { + id: t.id, + task_id: task_id_for_execute.clone(), + task_payload: out_bytes, + payload: Vec::new(), + }; + Some((solved, publish_payload_lock)) + }) + .unzip::<_, _, Vec<_>, Vec<_>>() + }) + .await; + + let (mut solved, publish_payload_locks) = match execute_result { + Ok(result) => result, + Err(err) => { + debug!( + "worker {}: execute failed for {}: {:?}", + worker_id, task_id, err + ); + continue; + } + }; if solved.is_empty() { continue; @@ -169,7 +206,7 @@ async fn worker_loop( } if let Err(status) = client - .publish_task_block(Request::new(proto::TaskBlock { tasks: solved })) + .publish_task_stream(Request::new(tokio_stream::iter(solved))) .await { debug!( @@ -238,10 +275,20 @@ async fn compute_module(api: Arc) { .map(|n| n.get()) .unwrap_or(4) }); + let compute_worker_count = std::env::var("GE_CLIENT_COMPUTE_WORKERS") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|v| *v > 0) + .unwrap_or(worker_count); + let _ = rayon::ThreadPoolBuilder::new() + .num_threads(compute_worker_count) + .thread_name(|idx| format!("engine-client-cpu-{idx}")) + .build_global(); info!( - "Starting {} client workers for {} task types", + "Starting {} client workers and {} compute workers for {} task types", worker_count, + compute_worker_count, task_ids.len() ); diff --git a/engine/src/lib.rs b/engine/src/lib.rs index 534ac0a..f2cff4b 100644 --- a/engine/src/lib.rs +++ b/engine/src/lib.rs @@ -6,8 +6,12 @@ use enginelib::{ events::{Events, ID}, task::{LeasedTask, StoredTask, StoredTaskBlock}, }; -use std::{collections::HashMap, sync::Arc}; -use tokio::sync::RwLock; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; +use tokio::sync::{RwLock, mpsc}; +use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status}; pub mod proto { @@ -51,6 +55,212 @@ impl EngineService { pub fn into_server(self) -> EngineServer { EngineServer::new(self) } + + async fn acquire_task_block_impl( + &self, + request: tonic::Request, + ) -> Result { + let challenge = get_auth(&request); + let task_id = request.get_ref().task_id.clone(); + let requested_block_size = request.get_ref().block_size; + let uid = get_uid(&request); + + { + let api = self.EngineAPI.read().await; + let db = api.db.clone(); + if !Events::CheckAuth(&api, uid.clone(), challenge, db) { + return Err(Status::permission_denied("Invalid authentication")); + }; + } + + let (namespace, task_name) = task_id.split_once(':').ok_or_else(|| { + Status::invalid_argument("Invalid task ID format, expected 'namespace:task'") + })?; + let key = ID(namespace, task_name); + + { + let api = self.EngineAPI.read().await; + if api.task_registry.get(&key).is_none() { + return Err(Status::invalid_argument("Task Does not Exist")); + } + if Events::ServerBeforeTaskAcquire(&api, uid.clone(), task_id.clone()) { + return Err(Status::aborted( + "Task acquire cancelled by server event handler", + )); + } + } + + let (receiver, lock_arc) = { + let api = self.EngineAPI.read().await; + let receiver = api + .task_queue + .tasks + .get(&key) + .map(|entry| entry.0.clone()) + .ok_or_else(|| Status::not_found("Unknown task type"))?; + let lock_arc = api.fill_locks.entry(key.clone()).or_default().clone(); + (receiver, lock_arc) + }; + + if receiver.is_empty() { + let _fill_guard = lock_arc.lock().await; + if receiver.is_empty() { + let api = self.EngineAPI.read().await; + ServerAPI::fill_queue(&api, key.clone(), requested_block_size); + } + } + + let block = receiver + .recv() + .await + .map_err(|_| Status::unavailable("Task queue closed"))?; + + { + let api = self.EngineAPI.read().await; + let mut entry = api.leased_tasks.tasks.entry(key.clone()).or_default(); + let now = Utc::now(); + let mut ids = Vec::with_capacity(block.tasks.len()); + for task in &block.tasks { + ids.push(task.id.clone()); + entry.push(LeasedTask { + stored_task: Arc::new(task.clone()), + user_id: uid.clone(), + given_at: now, + }); + } + drop(entry); + api.clear_active_ids(&key, &ids); + Events::ServerTaskBlockAcquired(&api, uid.clone(), task_id.clone(), ids); + } + + let tasks = block + .tasks + .into_iter() + .map(|t| proto::Task { + id: t.id, + task_id: task_id.clone(), + task_payload: t.bytes, + payload: Vec::new(), + }) + .collect(); + + Ok(proto::TaskBlock { tasks }) + } + + async fn publish_tasks_impl( + &self, + uid: String, + challenge: String, + tasks: Vec, + ) -> Result<(), Status> { + if tasks.is_empty() { + return Ok(()); + } + + let api = self.EngineAPI.read().await; + + { + let db = api.db.clone(); + if !Events::CheckAuth(&api, uid.clone(), challenge, db) { + return Err(Status::permission_denied("Invalid authentication")); + }; + } + + let mut groups: HashMap> = HashMap::new(); + for t in tasks { + let Some((ns, name)) = t.task_id.split_once(':') else { + info!("publish: skipping malformed task_id {}", t.task_id); + continue; + }; + groups + .entry((ns.to_string(), name.to_string())) + .or_default() + .push(t); + } + + for (key, tasks) in groups { + let task_id_str = format!("{}:{}", key.0, key.1); + let Some(reg_tsk) = api.task_registry.get(&key) else { + info!("publish: unknown task {}:{}, skipping group", key.0, key.1); + continue; + }; + + let mut candidates: Vec<(proto::Task, Vec)> = Vec::with_capacity(tasks.len()); + let mut candidate_ids: HashSet = HashSet::with_capacity(tasks.len()); + let mut published_ids: Vec = Vec::with_capacity(tasks.len()); + + for t in tasks { + let payload_for_event = Arc::new(std::sync::RwLock::new(t.task_payload.clone())); + if Events::ServerBeforeTaskPublish( + &api, + uid.clone(), + task_id_str.clone(), + t.id.clone(), + payload_for_event.clone(), + ) { + info!("publish: handler cancelled {}:{}", task_id_str, t.id); + continue; + } + + let payload = match payload_for_event.read() { + Ok(p) => p.clone(), + Err(_) => { + info!("publish: payload lock poisoned for {}", t.id); + continue; + } + }; + + if !reg_tsk.clone().verify(payload.clone()) { + info!("publish: verify failed for {}", t.id); + continue; + } + + candidate_ids.insert(t.id.clone()); + candidates.push((t, payload)); + } + + let mut removed_ids: HashSet = HashSet::with_capacity(candidates.len()); + { + let mut leased = api.leased_tasks.tasks.entry(key.clone()).or_default(); + leased.retain(|lease| { + if lease.user_id == uid + && candidate_ids.contains(&lease.stored_task.id) + && !removed_ids.contains(&lease.stored_task.id) + { + removed_ids.insert(lease.stored_task.id.clone()); + false + } else { + true + } + }); + } + + for (t, payload) in candidates { + if !removed_ids.remove(&t.id) { + info!("publish: no lease for {} held by {}", t.id, uid); + continue; + } + let stored = StoredTask { + id: t.id.clone(), + bytes: payload, + }; + if let Err(e) = api.put_solved(&key, &stored) { + info!("publish: sled put_solved failed for {}: {}", t.id, e); + continue; + } + if let Err(e) = api.delete_queued(&key, &t.id) { + info!("publish: sled delete_queued failed for {}: {}", t.id, e); + } + published_ids.push(t.id); + } + + if !published_ids.is_empty() { + Events::ServerTaskBlockPublished(&api, uid.clone(), task_id_str, published_ids); + } + } + + Ok(()) + } } #[tonic::async_trait] @@ -303,90 +513,27 @@ impl Engine for EngineService { &self, request: tonic::Request, ) -> Result, tonic::Status> { - let challenge = get_auth(&request); - let task_id = request.get_ref().task_id.clone(); - let uid = get_uid(&request); - - { - let api = self.EngineAPI.read().await; - let db = api.db.clone(); - if !Events::CheckAuth(&api, uid.clone(), challenge, db) { - return Err(Status::permission_denied("Invalid authentication")); - }; - } - - let (namespace, task_name) = task_id.split_once(':').ok_or_else(|| { - Status::invalid_argument("Invalid task ID format, expected 'namespace:task'") - })?; - let key = ID(namespace, task_name); - - { - let api = self.EngineAPI.read().await; - if api.task_registry.get(&key).is_none() { - return Err(Status::invalid_argument("Task Does not Exist")); - } - if Events::ServerBeforeTaskAcquire(&api, uid.clone(), task_id.clone()) { - return Err(Status::aborted( - "Task acquire cancelled by server event handler", - )); - } - } - - let receiver = { - let api = self.EngineAPI.read().await; - api.task_queue - .tasks - .get(&key) - .map(|entry| entry.0.clone()) - .ok_or_else(|| Status::not_found("Unknown task type"))? - }; + self.acquire_task_block_impl(request) + .await + .map(Response::new) + } - if receiver.is_empty() { - let lock_arc = { - let api = self.EngineAPI.read().await; - api.fill_locks.entry(key.clone()).or_default().clone() - }; - let _g = lock_arc.lock().await; - if receiver.is_empty() { - let api = self.EngineAPI.read().await; - ServerAPI::fill_queue(&api, key.clone()); - } - } + type AquireTaskStreamStream = ReceiverStream>; - let block = receiver - .recv() - .await - .map_err(|_| Status::unavailable("Task queue closed"))?; + async fn aquire_task_stream( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let block = self.acquire_task_block_impl(request).await?; + let (tx, rx) = mpsc::channel(block.tasks.len().max(1)); - { - let api = self.EngineAPI.read().await; - let mut entry = api.leased_tasks.tasks.entry(key.clone()).or_default(); - let now = Utc::now(); - let mut ids = Vec::with_capacity(block.tasks.len()); - for task in &block.tasks { - ids.push(task.id.clone()); - entry.push(LeasedTask { - stored_task: Arc::new(task.clone()), - user_id: uid.clone(), - given_at: now, - }); + for task in block.tasks { + if tx.send(Ok(task)).await.is_err() { + break; } - drop(entry); - Events::ServerTaskBlockAcquired(&api, uid.clone(), task_id.clone(), ids); } - let tasks = block - .tasks - .into_iter() - .map(|t| proto::Task { - id: t.id, - task_id: task_id.clone(), - task_payload: t.bytes, - payload: Vec::new(), - }) - .collect(); - - Ok(tonic::Response::new(proto::TaskBlock { tasks })) + Ok(Response::new(ReceiverStream::new(rx))) } async fn publish_task_block( @@ -395,90 +542,34 @@ impl Engine for EngineService { ) -> Result, tonic::Status> { let challenge = get_auth(&request); let uid = get_uid(&request); - let api = self.EngineAPI.read().await; - - { - let db = api.db.clone(); - if !Events::CheckAuth(&api, uid.clone(), challenge, db) { - return Err(Status::permission_denied("Invalid authentication")); - }; - } - - let mut groups: HashMap> = HashMap::new(); - for t in request.into_inner().tasks { - let Some((ns, name)) = t.task_id.split_once(':') else { - info!("publish: skipping malformed task_id {}", t.task_id); - continue; - }; - groups - .entry((ns.to_string(), name.to_string())) - .or_default() - .push(t); - } - - for (key, tasks) in groups { - let task_id_str = format!("{}:{}", key.0, key.1); - let Some(reg_tsk) = api.task_registry.get(&key) else { - info!("publish: unknown task {}:{}, skipping group", key.0, key.1); - continue; - }; - - let mut published_ids: Vec = Vec::with_capacity(tasks.len()); - - for t in tasks { - let payload_for_event = Arc::new(std::sync::RwLock::new(t.task_payload.clone())); - if Events::ServerBeforeTaskPublish( - &api, - uid.clone(), - task_id_str.clone(), - t.id.clone(), - payload_for_event.clone(), - ) { - info!("publish: handler cancelled {}:{}", task_id_str, t.id); - continue; - } - - let payload = match payload_for_event.read() { - Ok(p) => p.clone(), - Err(_) => { - info!("publish: payload lock poisoned for {}", t.id); - continue; - } - }; - - if !reg_tsk.clone().verify(payload.clone()) { - info!("publish: verify failed for {}", t.id); - continue; - } - - let mut leased = api.leased_tasks.tasks.entry(key.clone()).or_default(); - let Some(idx) = leased - .iter() - .position(|l| l.stored_task.id == t.id && l.user_id == uid) - else { - info!("publish: no lease for {} held by {}", t.id, uid); - continue; - }; - leased.remove(idx); - drop(leased); + self.publish_tasks_impl(uid, challenge, request.into_inner().tasks) + .await?; + Ok(tonic::Response::new(proto::Empty {})) + } - let stored = StoredTask { - id: t.id.clone(), - bytes: payload, - }; - if let Err(e) = api.put_solved(&key, &stored) { - info!("publish: sled put_solved failed for {}: {}", t.id, e); - continue; - } - if let Err(e) = api.delete_queued(&key, &t.id) { - info!("publish: sled delete_queued failed for {}: {}", t.id, e); - } - published_ids.push(t.id); + async fn publish_task_stream( + &self, + request: tonic::Request>, + ) -> Result, tonic::Status> { + let challenge = get_auth(&request); + let uid = get_uid(&request); + let batch_size = { + let api = self.EngineAPI.read().await; + api.cfg.config_toml.task_block_size.max(1) as usize + }; + let mut stream = request.into_inner(); + let mut batch = Vec::with_capacity(batch_size.min(1024)); + + while let Some(task) = stream.message().await? { + batch.push(task); + if batch.len() >= batch_size { + self.publish_tasks_impl(uid.clone(), challenge.clone(), std::mem::take(&mut batch)) + .await?; } + } - if !published_ids.is_empty() { - Events::ServerTaskBlockPublished(&api, uid.clone(), task_id_str, published_ids); - } + if !batch.is_empty() { + self.publish_tasks_impl(uid, challenge, batch).await?; } Ok(tonic::Response::new(proto::Empty {})) @@ -526,11 +617,22 @@ impl Engine for EngineService { task_id_str ))); }; + let enqueue_to_memory = api + .task_queue + .tasks + .get(&key) + .map(|channel| { + channel.1.len() < api.cfg.config_toml.task_queue_size.max(1) as usize + }) + .unwrap_or(false); let mut block_tasks: Vec = Vec::with_capacity(tasks.len()); let mut instance_ids: Vec = Vec::with_capacity(tasks.len()); + let mut active_ids: Vec = Vec::with_capacity(tasks.len()); let mut payloads: Vec>>> = Vec::with_capacity(tasks.len()); + let mut created_for_group: Vec = Vec::with_capacity(tasks.len()); + let mut stored_for_group: Vec = Vec::with_capacity(tasks.len()); for t in tasks { let payload_for_event = Arc::new(std::sync::RwLock::new(t.task_payload.clone())); @@ -557,24 +659,42 @@ impl Engine for EngineService { id: druid::Druid::default().to_hex(), bytes: payload, }; - if let Err(e) = api.put_queued(&key, &stored) { - info!("create: sled put_queued failed for {}: {}", stored.id, e); - continue; + if enqueue_to_memory { + api.mark_active_id(&key, stored.id.clone()); + active_ids.push(stored.id.clone()); } instance_ids.push(stored.id.clone()); payloads.push(payload_for_event); - created.push(proto::Task { + created_for_group.push(proto::Task { id: stored.id.clone(), task_id: task_id_str.clone(), task_payload: stored.bytes.clone(), payload: Vec::new(), }); - block_tasks.push(stored); + if enqueue_to_memory { + block_tasks.push(stored.clone()); + } + stored_for_group.push(stored); } - if !block_tasks.is_empty() { + if !instance_ids.is_empty() { + if let Err(e) = api.put_queued_batch(&key, &stored_for_group) { + api.clear_active_ids(&key, &active_ids); + info!("create: sled batch failed for {}: {}", task_id_str, e); + continue; + } + + created.extend(created_for_group); if let Some(channel) = api.task_queue.tasks.get(&key) { - let _ = channel.1.try_send(StoredTaskBlock { tasks: block_tasks }); + if enqueue_to_memory + && !block_tasks.is_empty() + && channel + .1 + .try_send(StoredTaskBlock { tasks: block_tasks }) + .is_err() + { + api.clear_active_ids(&key, &active_ids); + } } Events::ServerTaskBlockCreated(&api, task_id_str, instance_ids, payloads); } diff --git a/engine/tests/task_streaming_tests.rs b/engine/tests/task_streaming_tests.rs new file mode 100644 index 0000000..9e8c179 --- /dev/null +++ b/engine/tests/task_streaming_tests.rs @@ -0,0 +1,180 @@ +use engine::{EngineService, proto}; +use enginelib::{ + Identifier, Registry, + api::ServerAPI, + task::{StoredTask, StoredTaskBlock, Task, Verifiable}, +}; +use std::{ + net::{Ipv4Addr, SocketAddr, SocketAddrV4}, + sync::Arc, + time::{Duration, Instant}, +}; +use tokio::{net::TcpListener, sync::RwLock}; +use tonic::{Request, transport::Server}; + +const TEST_NS: &str = "stream_test"; +const TEST_TASK: &str = "noop"; + +#[derive(Debug, Clone, Default)] +struct NoopTask; + +impl Verifiable for NoopTask { + fn verify(&self, _b: Vec) -> bool { + true + } +} + +impl Task for NoopTask { + fn get_id(&self) -> Identifier { + (TEST_NS.to_string(), TEST_TASK.to_string()) + } + + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } + + fn run_cpu(&mut self) {} + + fn to_bytes(&self) -> Vec { + Vec::new() + } + + fn from_bytes(&self, _bytes: &[u8]) -> Box { + Box::new(self.clone()) + } + + fn from_toml(&self, _d: String) -> Box { + Box::new(self.clone()) + } + + fn to_toml(&self) -> String { + String::new() + } +} + +async fn pick_port() -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + listener.local_addr().unwrap().port() +} + +async fn build_client(port: u16) -> proto::engine_client::EngineClient { + let url = format!("http://127.0.0.1:{port}"); + let deadline = Instant::now() + Duration::from_secs(5); + + loop { + let endpoint = tonic::transport::Endpoint::try_from(url.clone()) + .unwrap() + .tcp_nodelay(true); + match endpoint.connect().await { + Ok(channel) => return proto::engine_client::EngineClient::new(channel), + Err(err) if Instant::now() < deadline => { + tokio::time::sleep(Duration::from_millis(20)).await; + let _ = err; + } + Err(err) => panic!("failed to connect to test server on {url}: {err}"), + } + } +} + +async fn spawn_test_server(port: u16) -> Arc> { + let mut api = ServerAPI::test_default(); + let id = (TEST_NS.to_string(), TEST_TASK.to_string()); + api.task_registry.register(Arc::new(NoopTask), id.clone()); + api.ensure_task_channel(id); + enginelib::event::register_inventory_handlers(&mut api); + + let api = Arc::new(RwLock::new(api)); + let engine = EngineService::new(api.clone()); + let addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, port)); + + tokio::spawn(async move { + let _ = Server::builder() + .add_service(engine.into_server()) + .serve(addr) + .await; + }); + + api +} + +#[tokio::test(flavor = "multi_thread")] +async fn acquire_and_publish_task_streams() { + let port = pick_port().await; + let api = spawn_test_server(port).await; + let mut client = build_client(port).await; + let task_id = format!("{TEST_NS}:{TEST_TASK}"); + + let created = client + .create_task_block(Request::new(proto::TaskBlock { + tasks: (0..3) + .map(|i| proto::Task { + id: String::new(), + task_id: task_id.clone(), + task_payload: vec![i], + payload: Vec::new(), + }) + .collect(), + })) + .await + .unwrap() + .into_inner(); + assert_eq!(created.tasks.len(), 3); + + let mut stream = client + .aquire_task_stream(Request::new(proto::TaskBlockRequest { + task_id: task_id.clone(), + block_size: 3, + })) + .await + .unwrap() + .into_inner(); + + let mut acquired = Vec::new(); + while let Some(task) = stream.message().await.unwrap() { + acquired.push(task); + } + assert_eq!(acquired.len(), 3); + + client + .publish_task_stream(Request::new(tokio_stream::iter(acquired))) + .await + .unwrap(); + + let key = (TEST_NS.to_string(), TEST_TASK.to_string()); + let api = api.read().await; + assert_eq!(api.scan_solved(&key).count(), 3); + assert_eq!(api.scan_queued(&key).count(), 0); +} + +#[test] +fn refill_skips_active_tasks_between_recv_and_lease() { + let mut api = ServerAPI::test_default(); + let key = (TEST_NS.to_string(), TEST_TASK.to_string()); + api.task_registry.register(Arc::new(NoopTask), key.clone()); + api.ensure_task_channel(key.clone()); + + let stored = StoredTask { + id: "active-task".to_string(), + bytes: vec![1], + }; + api.mark_active_id(&key, stored.id.clone()); + api.put_queued(&key, &stored).unwrap(); + + let (receiver, sender) = { + let channel = api.task_queue.tasks.get(&key).unwrap(); + (channel.0.clone(), channel.1.clone()) + }; + sender + .try_send(StoredTaskBlock { + tasks: vec![stored], + }) + .unwrap(); + + let received = receiver.try_recv().unwrap(); + assert_eq!(received.tasks.len(), 1); + assert!(receiver.is_empty()); + + ServerAPI::fill_queue(&api, key, 1); + + assert!(receiver.try_recv().is_err()); +} diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index 11b04f4..3dc5510 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -21,15 +21,16 @@ use std::{ }; pub struct ServerAPI { - pub cfg: Config, // RW - pub task_queue: TaskQueue, // RW - pub leased_tasks: LeasedTaskQueue, // RW + pub cfg: Config, // RW + pub task_queue: TaskQueue, // RW + pub leased_tasks: LeasedTaskQueue, // RW + pub active_task_ids: DashMap>, pub task_registry: EngineTaskRegistry, // RW pub event_bus: EventBus, // RW pub db: sled::Db, // R pub lib_manager: LibraryManager, // RW - // Serializes fill_queue() calls per Identifier so concurrent acquires can't - // both refill an empty channel and double-enqueue the same StoredTaskBlock. + // Serializes only sled refills per Identifier. Active task ids keep refills + // from duplicating tasks already queued in memory or between recv+lease. pub fill_locks: DashMap>>, } @@ -47,23 +48,31 @@ impl Default for ServerAPI { }, }, leased_tasks: LeasedTaskQueue::default(), + active_task_ids: DashMap::new(), fill_locks: DashMap::new(), } } } impl ServerAPI { - pub fn test_default() -> Self { - // `sled::Config::temporary(true)` defaults to `/dev/shm` on Linux when no path is set. - // Some environments deny writes there, so force a unique temp path. + fn temporary_db_path(prefix: &str) -> std::path::PathBuf { use std::sync::atomic::{AtomicUsize, Ordering}; static DB_COUNTER: AtomicUsize = AtomicUsize::new(0); let db_id = DB_COUNTER.fetch_add(1, Ordering::Relaxed); - let db_path = std::env::temp_dir().join(format!( - "enginelib-test-db-{}-{}", - std::process::id(), - db_id - )); + let root = std::env::var_os("ENGINE_TEST_DB_ROOT") + .map(std::path::PathBuf::from) + .unwrap_or_else(|| { + std::env::current_dir() + .unwrap_or_else(|_| std::env::temp_dir()) + .join("target") + .join("tmp") + }); + let _ = std::fs::create_dir_all(&root); + root.join(format!("{}-{}-{}", prefix, std::process::id(), db_id)) + } + + pub fn test_default() -> Self { + let db_path = Self::temporary_db_path("enginelib-test-db"); Self { cfg: Config::new(), @@ -83,6 +92,7 @@ impl ServerAPI { }, }, fill_locks: DashMap::new(), + active_task_ids: DashMap::new(), } } /// Ensure the task_queue + leased_tasks entries exist for this Identifier. @@ -92,7 +102,8 @@ impl ServerAPI { let (s, r) = async_channel::unbounded(); (r, s) }); - self.leased_tasks.tasks.entry(id).or_default(); + self.leased_tasks.tasks.entry(id.clone()).or_default(); + self.active_task_ids.entry(id).or_default(); } pub fn init(api: &mut Self) { @@ -106,6 +117,7 @@ impl ServerAPI { let (s, r) = async_channel::unbounded(); api.task_queue.tasks.entry(id.clone()).insert((r, s)); api.leased_tasks.tasks.entry(id.clone()).or_default(); + api.active_task_ids.entry(id.clone()).or_default(); } Self::init_events(api); @@ -114,14 +126,7 @@ impl ServerAPI { /// Client-side ServerAPI with a temp sled (client doesn't use db/task_queue /// at all; the temp dir is just to satisfy the struct field). pub fn default_client() -> Self { - use std::sync::atomic::{AtomicUsize, Ordering}; - static DB_COUNTER: AtomicUsize = AtomicUsize::new(0); - let db_id = DB_COUNTER.fetch_add(1, Ordering::Relaxed); - let db_path = std::env::temp_dir().join(format!( - "enginelib-client-db-{}-{}", - std::process::id(), - db_id - )); + let db_path = Self::temporary_db_path("enginelib-client-db"); Self { cfg: Config::new(), task_queue: TaskQueue::default(), @@ -140,6 +145,7 @@ impl ServerAPI { .unwrap(), lib_manager: LibraryManager::default(), fill_locks: DashMap::new(), + active_task_ids: DashMap::new(), } } @@ -193,6 +199,16 @@ impl ServerAPI { Ok(()) } + pub fn put_queued_batch(&self, task_id: &Identifier, tasks: &[StoredTask]) -> sled::Result<()> { + let mut batch = sled::Batch::default(); + for task in tasks { + let bytes = postcard::to_allocvec(task) + .map_err(|e| sled::Error::Unsupported(format!("postcard: {e}")))?; + batch.insert(Self::task_key(task_id, &task.id), bytes); + } + self.db.apply_batch(batch) + } + pub fn put_solved(&self, task_id: &Identifier, task: &StoredTask) -> sled::Result<()> { let bytes = postcard::to_allocvec(task) .map_err(|e| sled::Error::Unsupported(format!("postcard: {e}")))?; @@ -222,8 +238,42 @@ impl ServerAPI { .filter_map(|(_, value)| postcard::from_bytes::(&value).ok()) } - pub fn fill_queue(api: &ServerAPI, task_id: Identifier) { - let max_block = api.cfg.config_toml.task_block_size.max(1) as usize; + pub fn mark_active_id(&self, task_id: &Identifier, id: String) { + self.active_task_ids + .entry(task_id.clone()) + .or_default() + .insert(id); + } + + pub fn mark_active_ids(&self, task_id: &Identifier, ids: &[String]) { + if ids.is_empty() { + return; + } + + let mut active = self.active_task_ids.entry(task_id.clone()).or_default(); + for id in ids { + active.insert(id.clone()); + } + } + + pub fn clear_active_ids(&self, task_id: &Identifier, ids: &[String]) { + if ids.is_empty() { + return; + } + + if let Some(mut active) = self.active_task_ids.get_mut(task_id) { + for id in ids { + active.remove(id); + } + } + } + + pub fn fill_queue(api: &ServerAPI, task_id: Identifier, requested_block_size: u32) { + let max_block = if requested_block_size > 0 { + requested_block_size as usize + } else { + api.cfg.config_toml.task_block_size.max(1) as usize + }; let max_queue = api.cfg.config_toml.task_queue_size as usize; let Some(channel) = api.task_queue.tasks.get(&task_id) else { @@ -245,6 +295,11 @@ impl ServerAPI { .get(&task_id) .map(|v| v.iter().map(|l| l.stored_task.id.clone()).collect()) .unwrap_or_default(); + let active: HashSet = api + .active_task_ids + .get(&task_id) + .map(|v| v.iter().cloned().collect()) + .unwrap_or_default(); let mut block: Vec = Vec::with_capacity(max_block); @@ -253,14 +308,23 @@ impl ServerAPI { let Ok(task) = postcard::from_bytes::(&value) else { continue; }; - if leased.contains(&task.id) { + let is_active = active.contains(&task.id) + || api + .active_task_ids + .get(&task_id) + .map(|ids| ids.contains(&task.id)) + .unwrap_or(false); + if leased.contains(&task.id) || is_active { continue; } block.push(task); if block.len() == max_block { let full = std::mem::replace(&mut block, Vec::with_capacity(max_block)); + let ids: Vec = full.iter().map(|task| task.id.clone()).collect(); + api.mark_active_ids(&task_id, &ids); if sender.try_send(StoredTaskBlock { tasks: full }).is_err() { + api.clear_active_ids(&task_id, &ids); return; // receiver dropped } if sender.len() >= max_queue { @@ -270,13 +334,18 @@ impl ServerAPI { } if !block.is_empty() { - let _ = sender.try_send(StoredTaskBlock { tasks: block }); + let ids: Vec = block.iter().map(|task| task.id.clone()).collect(); + api.mark_active_ids(&task_id, &ids); + if sender.try_send(StoredTaskBlock { tasks: block }).is_err() { + api.clear_active_ids(&task_id, &ids); + } } } fn init_db(api: &mut ServerAPI) { api.task_queue = TaskQueue::default(); api.leased_tasks = LeasedTaskQueue::default(); + api.active_task_ids = DashMap::new(); } pub fn setup_logger() { From d456b2c7410103e0efbcf3dbf5e31fbb1e99e78d Mon Sep 17 00:00:00 2001 From: Styly Date: Sat, 18 Jul 2026 13:27:22 +0100 Subject: [PATCH 18/28] rewrite: clean --- Cargo.lock | 774 +---------------- engine/Cargo.toml | 29 +- engine/benches/engine_tps.rs | 1 - engine/build.rs | 11 - engine/proto/engine.proto | 76 -- engine/src/bin/bench.rs | 603 ------------- engine/src/bin/client.rs | 308 ------- engine/src/bin/packer.rs | 1068 ----------------------- engine/src/bin/server.rs | 46 +- engine/src/lib.rs | 705 --------------- engine/tests/task_streaming_tests.rs | 180 ---- enginelib/Cargo.toml | 6 +- enginelib/src/api.rs | 406 +-------- enginelib/src/config.rs | 27 - enginelib/src/event.rs | 8 + enginelib/src/events/cgrpc_event.rs | 2 +- enginelib/src/events/mod.rs | 2 +- enginelib/src/task.rs | 37 +- enginelib/tests/event_tests.rs | 221 ----- rfc/rfc1006.md | 1176 ++++++++++++++++++++++++++ 20 files changed, 1207 insertions(+), 4479 deletions(-) delete mode 100644 engine/benches/engine_tps.rs delete mode 100644 engine/build.rs delete mode 100644 engine/proto/engine.proto delete mode 100644 engine/src/bin/bench.rs delete mode 100644 engine/src/bin/client.rs delete mode 100644 engine/src/bin/packer.rs delete mode 100644 engine/src/lib.rs delete mode 100644 engine/tests/task_streaming_tests.rs delete mode 100644 enginelib/tests/event_tests.rs create mode 100644 rfc/rfc1006.md diff --git a/Cargo.lock b/Cargo.lock index 240d8d2..0aecd68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,56 +26,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anstream" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" - -[[package]] -name = "anstyle-parse" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys", -] - [[package]] name = "anyhow" version = "1.0.102" @@ -103,17 +53,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "async-trait" -version = "0.1.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "atomic-polyfill" version = "1.0.3" @@ -123,67 +62,12 @@ dependencies = [ "critical-section", ] -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" -[[package]] -name = "axum" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" -dependencies = [ - "axum-core", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "serde_core", - "sync_wrapper", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "sync_wrapper", - "tower-layer", - "tower-service", -] - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - [[package]] name = "bitflags" version = "1.3.2" @@ -283,17 +167,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "chacha20" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "rand_core", -] - [[package]] name = "chrono" version = "0.4.44" @@ -308,55 +181,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "clap" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", -] - -[[package]] -name = "clap_complete" -version = "4.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff7a1dccbdd8b078c2bdebff47e404615151534d5043da397ec50286816f9cb" -dependencies = [ - "clap", -] - -[[package]] -name = "clap_derive" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "clap_lex" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" - [[package]] name = "clru" version = "0.6.3" @@ -375,21 +199,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "colorchoice" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" - -[[package]] -name = "colored" -version = "3.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" -dependencies = [ - "windows-sys", -] - [[package]] name = "concurrent-queue" version = "2.5.0" @@ -414,15 +223,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cpufeatures" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" -dependencies = [ - "libc", -] - [[package]] name = "crc32fast" version = "1.5.0" @@ -624,27 +424,12 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "druid" -version = "0.1.0" -source = "git+https://github.com/GrandEngineering/druid.git#dc2e5dcab0568079ffb4bad9a58e489c51c7fbad" -dependencies = [ - "rand", - "uuid", -] - [[package]] name = "dunce" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" -[[package]] -name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" - [[package]] name = "embedded-io" version = "0.4.0" @@ -670,23 +455,7 @@ dependencies = [ name = "engine" version = "0.1.0" dependencies = [ - "clap", - "clap_complete", - "colored", - "druid", "enginelib", - "prost", - "rayon", - "serde", - "sha2", - "tokio", - "tokio-stream", - "toml", - "tonic", - "tonic-build", - "tonic-prost", - "tonic-prost-build", - "tonic-reflection", ] [[package]] @@ -783,12 +552,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" -[[package]] -name = "fixedbitset" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" - [[package]] name = "fnv" version = "1.0.7" @@ -817,44 +580,11 @@ dependencies = [ "winapi", ] -[[package]] -name = "futures-channel" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" -dependencies = [ - "futures-core", -] - [[package]] name = "futures-core" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" - -[[package]] -name = "futures-sink" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" - -[[package]] -name = "futures-task" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" - -[[package]] -name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" -dependencies = [ - "futures-core", - "futures-task", - "pin-project-lite", - "slab", -] +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "fxhash" @@ -895,7 +625,6 @@ dependencies = [ "cfg-if", "libc", "r-efi", - "rand_core", "wasip2", "wasip3", ] @@ -1639,25 +1368,6 @@ dependencies = [ "gix-validate", ] -[[package]] -name = "h2" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "hash32" version = "0.2.1" @@ -1738,106 +1448,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "http" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" -dependencies = [ - "bytes", - "itoa", -] - -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "hyper" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" -dependencies = [ - "atomic-waker", - "bytes", - "futures-channel", - "futures-core", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-timeout" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" -dependencies = [ - "hyper", - "hyper-util", - "pin-project-lite", - "tokio", - "tower-service", -] - -[[package]] -name = "hyper-util" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "http", - "http-body", - "hyper", - "libc", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", -] - [[package]] name = "iana-time-zone" version = "0.1.65" @@ -1913,21 +1523,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "is_terminal_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" - -[[package]] -name = "itertools" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.18" @@ -2073,12 +1668,6 @@ dependencies = [ "regex-automata", ] -[[package]] -name = "matchit" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" - [[package]] name = "maybe-async" version = "0.2.10" @@ -2105,12 +1694,6 @@ dependencies = [ "libc", ] -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - [[package]] name = "mio" version = "1.2.0" @@ -2122,12 +1705,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "multimap" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" - [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2167,12 +1744,6 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" - [[package]] name = "option-ext" version = "0.2.0" @@ -2249,37 +1820,6 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "petgraph" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" -dependencies = [ - "fixedbitset", - "hashbrown 0.15.5", - "indexmap", -] - -[[package]] -name = "pin-project" -version = "1.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "pin-project-lite" version = "0.2.17" @@ -2354,79 +1894,6 @@ dependencies = [ "parking_lot 0.12.5", ] -[[package]] -name = "prost" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" -dependencies = [ - "heck", - "itertools", - "log", - "multimap", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "pulldown-cmark", - "pulldown-cmark-to-cmark", - "regex", - "syn", - "tempfile", -] - -[[package]] -name = "prost-derive" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "prost-types" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" -dependencies = [ - "prost", -] - -[[package]] -name = "pulldown-cmark" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c3a14896dfa883796f1cb410461aef38810ea05f2b2c33c5aded3649095fdad" -dependencies = [ - "bitflags 2.11.1", - "memchr", - "unicase", -] - -[[package]] -name = "pulldown-cmark-to-cmark" -version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" -dependencies = [ - "pulldown-cmark", -] - [[package]] name = "quote" version = "1.0.45" @@ -2442,43 +1909,6 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" -dependencies = [ - "chacha20", - "getrandom 0.4.2", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" - -[[package]] -name = "rayon" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - [[package]] name = "redox_syscall" version = "0.2.16" @@ -2658,7 +2088,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures 0.2.17", + "cpufeatures", "digest", ] @@ -2672,17 +2102,6 @@ dependencies = [ "sha1", ] -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest", -] - [[package]] name = "sharded-slab" version = "0.1.7" @@ -2734,12 +2153,6 @@ dependencies = [ "libc", ] -[[package]] -name = "slab" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" - [[package]] name = "sled" version = "0.34.7" @@ -2810,12 +2223,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" - [[package]] name = "tar" version = "0.4.45" @@ -2945,30 +2352,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-stream" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - [[package]] name = "toml" version = "1.1.2+spec-1.1.0" @@ -3008,119 +2391,6 @@ version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" -[[package]] -name = "tonic" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" -dependencies = [ - "async-trait", - "axum", - "base64", - "bytes", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-timeout", - "hyper-util", - "percent-encoding", - "pin-project", - "socket2", - "sync_wrapper", - "tokio", - "tokio-stream", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tonic-build" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1882ac3bf5ef12877d7ed57aad87e75154c11931c2ba7e6cde5e22d63522c734" -dependencies = [ - "prettyplease", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tonic-prost" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" -dependencies = [ - "bytes", - "prost", - "tonic", -] - -[[package]] -name = "tonic-prost-build" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3144df636917574672e93d0f56d7edec49f90305749c668df5101751bb8f95a" -dependencies = [ - "prettyplease", - "proc-macro2", - "prost-build", - "prost-types", - "quote", - "syn", - "tempfile", - "tonic-build", -] - -[[package]] -name = "tonic-reflection" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf0685a51e6d02b502ba0764002e766b7f3042aed13d9234925b6ffbfa3fca7" -dependencies = [ - "prost", - "prost-types", - "tokio", - "tokio-stream", - "tonic", - "tonic-prost", -] - -[[package]] -name = "tower" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" -dependencies = [ - "futures-core", - "futures-util", - "indexmap", - "pin-project-lite", - "slab", - "sync_wrapper", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - [[package]] name = "tracing" version = "0.1.44" @@ -3203,24 +2473,12 @@ dependencies = [ "syn", ] -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - [[package]] name = "typenum" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" -[[package]] -name = "unicase" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" - [[package]] name = "unicode-bom" version = "2.0.3" @@ -3248,23 +2506,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - -[[package]] -name = "uuid" -version = "1.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" -dependencies = [ - "getrandom 0.4.2", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "valuable" version = "0.1.1" @@ -3329,15 +2570,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" diff --git a/engine/Cargo.toml b/engine/Cargo.toml index 9bc39c9..865e27a 100644 --- a/engine/Cargo.toml +++ b/engine/Cargo.toml @@ -9,32 +9,5 @@ description = "A Blazingly fast distributed task system" default = [] dev = [] [dependencies] -clap = { version = "4.5.60", features = ["derive"] } -clap_complete = "4.5.66" -colored = "3.1.1" -# directories = "5.0.1" -druid = { git = "https://github.com/GrandEngineering/druid.git" } -enginelib = { path = "../enginelib" } -# libloading = "0.8.6" -prost = "0.14" -rayon = "1.10" -serde = { workspace = true } -sha2 = "0.10" -# serde = "1.0.219" -tokio = { version = "1.50.0", features = ["rt-multi-thread", "macros"] } -tokio-stream = "0.1.17" -toml = { workspace = true } -# toml = "0.8.19" -tonic = "0.14" -tonic-prost = "0.14.5" - -tonic-reflection = "0.14" -[[bench]] -name = "engine_tps" -path = "benches/engine_tps.rs" -harness = false - -[build-dependencies] -tonic-build = "0.14" -tonic-prost-build = "0.14" +enginelib = { path = "../enginelib" } diff --git a/engine/benches/engine_tps.rs b/engine/benches/engine_tps.rs deleted file mode 100644 index 96df685..0000000 --- a/engine/benches/engine_tps.rs +++ /dev/null @@ -1 +0,0 @@ -include!("../src/bin/bench.rs"); diff --git a/engine/build.rs b/engine/build.rs deleted file mode 100644 index 85f99ac..0000000 --- a/engine/build.rs +++ /dev/null @@ -1,11 +0,0 @@ -use std::error::Error; -use std::{env, path::PathBuf}; -fn main() -> Result<(), Box> { - let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); - - tonic_prost_build::configure() - .file_descriptor_set_path(out_dir.join("engine_descriptor.bin")) - .compile_protos(&["proto/engine.proto"], &["proto"])?; - - Ok(()) -} diff --git a/engine/proto/engine.proto b/engine/proto/engine.proto deleted file mode 100644 index bad8a33..0000000 --- a/engine/proto/engine.proto +++ /dev/null @@ -1,76 +0,0 @@ -syntax = "proto3"; -package engine; -service Engine { - rpc AquireTaskBlock(TaskBlockRequest) returns (TaskBlock); - rpc AquireTaskStream(TaskBlockRequest) returns (stream Task); - rpc AquireTaskReg(empty) returns (TaskRegistry); - rpc PublishTaskBlock(TaskBlock) returns (empty); - rpc PublishTaskStream(stream Task) returns (empty); - rpc cgrpc(cgrpcmsg) returns (cgrpcmsg); - rpc CreateTaskBlock(TaskBlock) returns (TaskBlock); - rpc DeleteTaskBlock(TaskSelector) returns (empty); - rpc GetTasks(TaskPageRequest) returns (TaskPage); - rpc CheckAuth(empty) returns (empty); - rpc GetMetadata(empty) returns (ServerMetadata); -} -message ServerMetadata { - string engine_api = 1; - repeated ModuleInfo mods = 2; -} -message ModuleInfo { - string mod_id = 1; - string api_version = 2; - string rustc_version = 3; - string mod_version = 4; -} -message TaskSelector { - TaskState state = 1; - string namespace = 2; - string task = 3; - string id = 4; -} -message empty {} -enum TaskState { - QUEUED = 0; - SOLVED = 1; - LEASED = 2; -} -message TaskPageRequest { - string namespace = 1; - string task = 2; - uint64 page = 3; - uint32 pageSize = 4; - TaskState state = 5; -} -message TaskPage { - string namespace = 1; - string task = 2; - uint64 page = 3; - uint32 pageSize = 4; - TaskState state = 5; - repeated Task tasks = 6; -} -message cgrpcmsg { - string handler_mod_id = 1; - string handler_id = 2; - bytes event_payload = 3; - string token = 4; -} - -message TaskRegistry { - repeated string tasks = 1; // namespace:task -} - -message TaskBlockRequest { - string task_id = 1; // namespace:task - uint32 block_size = 2; -} -message Task { - string id = 4; // the task unique identifier - bytes task_payload = 1; - string task_id = 2; // namespace:task - bytes payload = 3; -} -message TaskBlock { - repeated Task tasks = 1; -} diff --git a/engine/src/bin/bench.rs b/engine/src/bin/bench.rs deleted file mode 100644 index 29a8ff0..0000000 --- a/engine/src/bin/bench.rs +++ /dev/null @@ -1,603 +0,0 @@ -// End-to-end TPS benchmark for the PoW task gRPC pipeline. -// -// Spins the engine server in-process on a localhost ephemeral port, registers a -// local copy of the engine_mod:pow task, then runs two phases over real loopback gRPC: -// 1. Enqueue N tasks via CreateTaskBlock in chunks -// 2. Drain via W concurrent workers (aquire_task_stream + publish_task_stream) -// Prints per-phase elapsed/TPS and a linear extrapolation to 1B tasks. - -use clap::Parser; -use engine::{EngineService, proto}; -use enginelib::{ - Identifier, Registry, - api::ServerAPI, - task::{Task, Verifiable}, -}; -use proto::engine_client; -use rayon::prelude::*; -use serde::{Deserialize, Serialize}; -use sha2::{Digest, Sha256}; -use std::{ - fs, - net::{Ipv4Addr, SocketAddr, SocketAddrV4}, - process::Command, - sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, - }, - time::{Duration, Instant}, -}; -use tokio::{ - net::TcpListener, - sync::{RwLock, watch}, - task::JoinSet, -}; -use tonic::{Request, transport::Server}; - -const BENCH_NS: &str = "engine_mod"; -const BENCH_TASK: &str = "pow"; - -fn parse_nonzero_usize(value: &str) -> Result { - let parsed = value - .parse::() - .map_err(|err| format!("expected positive integer: {err}"))?; - if parsed == 0 { - Err("must be greater than zero".to_string()) - } else { - Ok(parsed) - } -} - -fn parse_nonzero_u32(value: &str) -> Result { - let parsed = value - .parse::() - .map_err(|err| format!("expected positive integer: {err}"))?; - if parsed == 0 { - Err("must be greater than zero".to_string()) - } else { - Ok(parsed) - } -} - -fn parse_nonzero_u64(value: &str) -> Result { - let parsed = value - .parse::() - .map_err(|err| format!("expected positive integer: {err}"))?; - if parsed == 0 { - Err("must be greater than zero".to_string()) - } else { - Ok(parsed) - } -} - -#[derive(Parser, Debug)] -#[command(name = "bench", about = "Engine block-based TPS bench")] -struct Args { - /// Total tasks to enqueue + process. - #[arg(long, default_value_t = 10_000_000, value_parser = parse_nonzero_usize)] - tasks: usize, - /// Worker count for the process phase. - #[arg(long, default_value_t = num_cpus_fallback(), value_parser = parse_nonzero_usize)] - workers: usize, - /// Rayon CPU worker count used to solve tasks inside acquired blocks. - #[arg(long, default_value_t = num_cpus_fallback(), value_parser = parse_nonzero_usize)] - compute_workers: usize, - /// Block size for enqueue (CreateTaskBlock chunks). - #[arg(long, default_value_t = 1024, value_parser = parse_nonzero_u32)] - block_size: u32, - /// PoW leading zero nibbles required by each task. - #[arg(long, default_value_t = 1)] - pow_zeros: u8, - /// First u32 seed to use when generating PoW task payloads. - #[arg(long, default_value_t = 0)] - seed_start: u32, - /// Per-acquire timeout so workers can recover if the queue unexpectedly stalls. - #[arg(long, default_value_t = 1000, value_parser = parse_nonzero_u64)] - acquire_timeout_ms: u64, - /// Legacy no-op benchmark argument retained for old command lines. - #[arg(long, hide = true)] - _payload_bytes: Option, - /// Cargo passes this to harness-free benchmark targets. - #[arg(long = "bench", hide = true, action = clap::ArgAction::SetTrue)] - _cargo_bench: bool, -} - -fn num_cpus_fallback() -> usize { - std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(4) -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -struct PowTask { - seed: u32, - zeros: u8, - nonce: u64, -} - -impl PowTask { - #[inline] - fn has_leading_zero_nibbles(digest: &[u8; 32], zeros: u8) -> bool { - let zeros = usize::from(zeros).min(64); - let full_zero_bytes = zeros / 2; - - if digest[..full_zero_bytes].iter().any(|&b| b != 0) { - return false; - } - - if zeros % 2 == 1 { - (digest[full_zero_bytes] & 0xF0) == 0 - } else { - true - } - } -} - -impl Verifiable for PowTask { - fn verify(&self, b: Vec) -> bool { - enginelib::api::postcard::from_bytes::(&b).is_ok() - } -} - -impl Task for PowTask { - fn get_id(&self) -> Identifier { - (BENCH_NS.to_string(), BENCH_TASK.to_string()) - } - - fn clone_box(&self) -> Box { - Box::new(self.clone()) - } - - fn run_cpu(&mut self) { - let zeros = self.zeros.min(64); - let seed_bytes = self.seed.to_le_bytes(); - - for n in 0..=u64::MAX { - let mut hasher = Sha256::new(); - hasher.update(n.to_le_bytes()); - hasher.update(seed_bytes); - let digest: [u8; 32] = hasher.finalize().into(); - - if Self::has_leading_zero_nibbles(&digest, zeros) { - self.nonce = n; - return; - } - } - } - - fn to_bytes(&self) -> Vec { - enginelib::api::to_allocvec(self).unwrap() - } - - fn from_bytes(&self, bytes: &[u8]) -> Box { - Box::new(enginelib::api::from_bytes::(bytes).unwrap()) - } - - fn from_toml(&self, d: String) -> Box { - Box::new(toml::from_str::(&d).unwrap()) - } - - fn to_toml(&self) -> String { - toml::to_string(self).unwrap() - } -} - -/// Pick a free localhost port via an ephemeral TcpListener bind+drop. -async fn pick_port() -> u16 { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - listener.local_addr().unwrap().port() -} - -async fn build_server_api(block_size: u32) -> Arc> { - let mut api = ServerAPI::test_default(); - api.cfg.config_toml.task_block_size = block_size; - let id: Identifier = (BENCH_NS.to_string(), BENCH_TASK.to_string()); - api.task_registry - .register(Arc::new(PowTask::default()), id.clone()); - api.ensure_task_channel(id); - // Wire up inventory event handlers — the default core::auth_event handler - // approves auth, which the bench client relies on (it sends no creds). - enginelib::event::register_inventory_handlers(&mut api); - Arc::new(RwLock::new(api)) -} - -async fn spawn_server(port: u16, block_size: u32) -> Arc> { - let api = build_server_api(block_size).await; - let engine = EngineService::new(api.clone()); - let addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), port)); - tokio::spawn(async move { - Server::builder() - .add_service(engine.into_server()) - .serve(addr) - .await - .ok(); - }); - api -} - -async fn build_client(port: u16) -> engine_client::EngineClient { - let url = format!("http://127.0.0.1:{}", port); - let deadline = Instant::now() + Duration::from_secs(5); - loop { - let endpoint = tonic::transport::Endpoint::try_from(url.clone()) - .unwrap() - .tcp_nodelay(true); - match endpoint.connect().await { - Ok(channel) => return engine_client::EngineClient::new(channel), - Err(err) if Instant::now() < deadline => { - tokio::time::sleep(Duration::from_millis(20)).await; - let _ = err; - } - Err(err) => panic!("failed to connect to benchmark server on {url}: {err}"), - } - } -} - -fn fmt_dur(d: Duration) -> String { - let secs = d.as_secs_f64(); - if secs >= 3600.0 { - format!("{:.2}h", secs / 3600.0) - } else if secs >= 60.0 { - format!("{:.2}m", secs / 60.0) - } else { - format!("{:.2}s", secs) - } -} - -fn fmt_tps(tps: f64) -> String { - if tps >= 1_000_000.0 { - format!("{:.2}M/s", tps / 1_000_000.0) - } else if tps >= 1_000.0 { - format!("{:.1}k/s", tps / 1_000.0) - } else { - format!("{:.0}/s", tps) - } -} - -fn clock_ticks_per_second() -> f64 { - Command::new("getconf") - .arg("CLK_TCK") - .output() - .ok() - .and_then(|output| String::from_utf8(output.stdout).ok()) - .and_then(|text| text.trim().parse::().ok()) - .filter(|ticks| *ticks > 0.0) - .unwrap_or(100.0) -} - -fn process_cpu_time() -> Option { - let stat = fs::read_to_string("/proc/self/stat").ok()?; - let fields = stat.rsplit_once(") ")?.1; - let values: Vec<&str> = fields.split_whitespace().collect(); - let utime = values.get(11)?.parse::().ok()?; - let stime = values.get(12)?.parse::().ok()?; - let secs = (utime + stime) as f64 / clock_ticks_per_second(); - Some(Duration::from_secs_f64(secs)) -} - -fn cpu_delta_since(start: Option) -> Option { - Some(process_cpu_time()?.saturating_sub(start?)) -} - -fn fmt_cpu(cpu: Option, wall: Duration) -> String { - let Some(cpu) = cpu else { - return "cpu n/a".to_string(); - }; - let pct = if wall.is_zero() { - 0.0 - } else { - cpu.as_secs_f64() / wall.as_secs_f64() * 100.0 - }; - format!("cpu {} ({pct:.0}%)", fmt_dur(cpu)) -} - -async fn enqueue_phase( - client: &mut engine_client::EngineClient, - total: usize, - block_size: u32, - zeros: u8, - seed_start: u32, -) -> Duration { - let task_id = format!("{}:{}", BENCH_NS, BENCH_TASK); - let start = Instant::now(); - let mut sent = 0usize; - while sent < total { - let chunk = (block_size as usize).min(total - sent); - let tasks: Vec = (0..chunk) - .map(|i| { - let task = PowTask { - seed: seed_start.wrapping_add((sent + i) as u32), - zeros, - nonce: 0, - }; - proto::Task { - id: String::new(), - task_id: task_id.clone(), - task_payload: task.to_bytes(), - payload: Vec::new(), - } - }) - .collect(); - client - .create_task_block(Request::new(proto::TaskBlock { tasks })) - .await - .expect("create_task_block failed"); - sent += chunk; - } - start.elapsed() -} - -#[derive(Debug)] -struct ProcessReport { - elapsed: Duration, - blocks_per_worker: Vec, -} - -fn solve_pow_proto_task(task: proto::Task) -> Result { - let mut pow = enginelib::api::from_bytes::(&task.task_payload) - .map_err(|err| format!("invalid PoW payload for {}: {err}", task.id))?; - pow.run_cpu(); - Ok(proto::Task { - id: task.id, - task_id: task.task_id, - task_payload: pow.to_bytes(), - payload: Vec::new(), - }) -} - -async fn process_phase( - port: u16, - total: usize, - workers: usize, - block_size: u32, - acquire_timeout: Duration, -) -> Result { - let task_id = format!("{}:{}", BENCH_NS, BENCH_TASK); - let counter = Arc::new(AtomicUsize::new(0)); - let blocks_per_worker = Arc::new( - (0..workers) - .map(|_| AtomicUsize::new(0)) - .collect::>(), - ); - let (done_tx, done_rx) = watch::channel(false); - let start = Instant::now(); - - let mut handles = JoinSet::new(); - for worker_id in 0..workers { - let task_id = task_id.clone(); - let counter = counter.clone(); - let blocks_per_worker = blocks_per_worker.clone(); - let done_tx = done_tx.clone(); - let mut done_rx = done_rx.clone(); - handles.spawn(async move { - let mut client = build_client(port).await; - loop { - if *done_rx.borrow() || counter.load(Ordering::Relaxed) >= total { - break; - } - // The server-side acquire waits on a queue receive. The timeout - // catches unexpected stalls, while the finished watch signal - // keeps normal completion from paying the full timeout tail. - let acquire = client.aquire_task_stream(Request::new(proto::TaskBlockRequest { - task_id: task_id.clone(), - block_size, - })); - let resp = tokio::select! { - changed = done_rx.changed() => { - if changed.is_ok() && *done_rx.borrow() { - break; - } - continue; - } - resp = tokio::time::timeout(acquire_timeout, acquire) => resp, - }; - let mut task_stream = match resp { - Ok(Ok(r)) => r.into_inner(), - Ok(Err(status)) => { - if counter.load(Ordering::Relaxed) >= total { - break; - } - if matches!( - status.code(), - tonic::Code::InvalidArgument | tonic::Code::PermissionDenied - ) { - return Err(format!("acquire failed for {task_id}: {status}")); - } - continue; - } - Err(_) => { - if counter.load(Ordering::Relaxed) >= total { - break; - } - continue; - } - }; - - let mut acquired = Vec::with_capacity(block_size as usize); - loop { - match task_stream.message().await { - Ok(Some(task)) => acquired.push(task), - Ok(None) => break, - Err(status) => { - return Err(format!("acquire stream failed for {task_id}: {status}")); - } - } - } - - if acquired.is_empty() { - continue; - } - let n = acquired.len(); - blocks_per_worker[worker_id].fetch_add(1, Ordering::Relaxed); - let solved: Vec = tokio::task::spawn_blocking(move || { - acquired - .into_par_iter() - .map(solve_pow_proto_task) - .collect::>() - }) - .await - .map_err(|err| format!("PoW worker task failed: {err}"))??; - match client - .publish_task_stream(Request::new(tokio_stream::iter(solved))) - .await - { - Ok(_) => { - let processed = counter.fetch_add(n, Ordering::Relaxed) + n; - if processed >= total { - let _ = done_tx.send(true); - break; - } - } - Err(status) => return Err(format!("publish failed for {task_id}: {status}")), - } - } - Ok::<(), String>(()) - }); - } - - while let Some(result) = handles.join_next().await { - match result { - Ok(Ok(())) => {} - Ok(Err(err)) => { - let _ = done_tx.send(true); - handles.abort_all(); - return Err(err); - } - Err(err) => { - let _ = done_tx.send(true); - handles.abort_all(); - return Err(format!("worker task failed: {err}")); - } - } - } - Ok(ProcessReport { - elapsed: start.elapsed(), - blocks_per_worker: blocks_per_worker - .iter() - .map(|count| count.load(Ordering::Relaxed)) - .collect(), - }) -} - -#[tokio::main(flavor = "multi_thread")] -async fn main() { - let args = Args::parse(); - let _ = rayon::ThreadPoolBuilder::new() - .num_threads(args.compute_workers) - .thread_name(|idx| format!("engine-bench-cpu-{idx}")) - .build_global(); - - println!( - "engine bench: task={}:{} tasks={} workers={} compute_workers={} block_size={} pow_zeros={} seed_start={}", - BENCH_NS, - BENCH_TASK, - args.tasks, - args.workers, - args.compute_workers, - args.block_size, - args.pow_zeros, - args.seed_start - ); - - let port = pick_port().await; - let _server_api = spawn_server(port, args.block_size).await; - let mut client = build_client(port).await; - - println!("Phase 1: enqueue"); - let enq_cpu_start = process_cpu_time(); - let enq = enqueue_phase( - &mut client, - args.tasks, - args.block_size, - args.pow_zeros, - args.seed_start, - ) - .await; - let enq_tps = args.tasks as f64 / enq.as_secs_f64(); - println!( - " {} tasks in {} ({}, {})", - args.tasks, - fmt_dur(enq), - fmt_tps(enq_tps), - fmt_cpu(cpu_delta_since(enq_cpu_start), enq) - ); - - println!("Phase 2: process"); - let proc_cpu_start = process_cpu_time(); - let proc_report = process_phase( - port, - args.tasks, - args.workers, - args.block_size, - Duration::from_millis(args.acquire_timeout_ms), - ) - .await - .expect("process phase failed"); - let proc = proc_report.elapsed; - let proc_tps = args.tasks as f64 / proc.as_secs_f64(); - println!( - " {} tasks in {} ({}, {})", - args.tasks, - fmt_dur(proc), - fmt_tps(proc_tps), - fmt_cpu(cpu_delta_since(proc_cpu_start), proc) - ); - let active_workers = proc_report - .blocks_per_worker - .iter() - .filter(|blocks| **blocks > 0) - .count(); - let total_blocks: usize = proc_report.blocks_per_worker.iter().sum(); - let min_nonzero_blocks = proc_report - .blocks_per_worker - .iter() - .copied() - .filter(|blocks| *blocks > 0) - .min() - .unwrap_or(0); - let max_blocks = proc_report - .blocks_per_worker - .iter() - .copied() - .max() - .unwrap_or(0); - println!( - " worker blocks: total={} active_workers={}/{} min_nonzero={} max={}", - total_blocks, active_workers, args.workers, min_nonzero_blocks, max_blocks - ); - - let total = enq + proc; - let total_tps = args.tasks as f64 / total.as_secs_f64(); - println!( - "Total wall: {} ({} end-to-end)", - fmt_dur(total), - fmt_tps(total_tps) - ); - - let scale = 1_000_000_000.0 / args.tasks as f64; - println!("\n1B linear extrapolation (naive — scaling caveats below):"); - println!( - " Enqueue: {} @ {}", - fmt_dur(enq.mul_f64(scale)), - fmt_tps(enq_tps) - ); - println!( - " Process: {} @ {}", - fmt_dur(proc.mul_f64(scale)), - fmt_tps(proc_tps) - ); - println!( - " End-to-end: {} @ {}", - fmt_dur(total.mul_f64(scale)), - fmt_tps(total_tps) - ); - - println!("\nCaveats: linear extrapolation assumes:"); - println!(" - sled scales linearly with prefix size (it doesn't past tens of GB)"); - println!( - " - zeros={} PoW difficulty stays representative", - args.pow_zeros - ); - println!(" - task_queue_size soft cap doesn't throttle producers"); - println!(" - loopback gRPC ≈ real network (it's typically 2-5x faster than LAN)"); -} diff --git a/engine/src/bin/client.rs b/engine/src/bin/client.rs deleted file mode 100644 index 0e04de0..0000000 --- a/engine/src/bin/client.rs +++ /dev/null @@ -1,308 +0,0 @@ -use enginelib::{ - Registry, api::ServerAPI, event::info, events::Events, plugin::LibraryInstance, prelude::debug, -}; -use proto::engine_client; -use rayon::prelude::*; -use std::{collections::HashMap, error::Error, sync::Arc}; -use tonic::{ - Request, - metadata::{MetadataKey, MetadataValue}, - transport::Endpoint, -}; - -pub mod proto { - tonic::include_proto!("engine"); -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - let mut api = ServerAPI::default_client(); - ServerAPI::init_client(&mut api); - Events::ClientStart(&api); - - compute_module(Arc::new(api)).await; - Ok(()) -} - -fn make_interceptor( - api_for_interceptor: Arc, -) -> impl FnMut(Request<()>) -> Result, tonic::Status> + Clone { - move |mut req: Request<()>| { - let headers = Arc::new(std::sync::RwLock::new(HashMap::::new())); - Events::ClientAuthPrepare(api_for_interceptor.as_ref(), headers.clone()); - - if let Ok(headers) = headers.read() { - for (key, value) in headers.iter() { - if let (Ok(key), Ok(value)) = ( - MetadataKey::from_bytes(key.as_bytes()), - MetadataValue::try_from(value.as_str()), - ) { - req.metadata_mut().insert(key, value); - } - } - } - - Ok(req) - } -} - -async fn worker_loop( - worker_id: usize, - api: Arc, - channel: tonic::transport::Channel, - task_ids: Arc>, -) { - let interceptor = make_interceptor(api.clone()); - let mut client = engine_client::EngineClient::with_interceptor(channel, interceptor); - - loop { - let mut got_any = false; - - for task_id in task_ids.iter() { - if Events::BeforeTaskBlockAcquire(api.as_ref(), vec![task_id.clone()]) { - continue; - } - - let resp = client - .aquire_task_stream(Request::new(proto::TaskBlockRequest { - task_id: task_id.clone(), - block_size: 0, - })) - .await; - - let mut task_stream = match resp { - Ok(r) => r.into_inner(), - Err(status) if status.code() == tonic::Code::NotFound => continue, - Err(status) if status.code() == tonic::Code::PermissionDenied => { - debug!( - "worker {}: auth failed during acquire for {}", - worker_id, task_id - ); - tokio::time::sleep(std::time::Duration::from_millis(200)).await; - continue; - } - Err(status) => { - debug!( - "worker {}: acquire failed for {}: {:?}", - worker_id, task_id, status - ); - continue; - } - }; - - let mut tasks = Vec::new(); - loop { - match task_stream.message().await { - Ok(Some(task)) => tasks.push(task), - Ok(None) => break, - Err(status) => { - debug!( - "worker {}: acquire stream failed for {}: {:?}", - worker_id, task_id, status - ); - tasks.clear(); - break; - } - } - } - - let block = proto::TaskBlock { tasks }; - if block.tasks.is_empty() { - continue; - } - got_any = true; - - let identifier = match task_id.split_once(':') { - Some(v) => (v.0.to_string(), v.1.to_string()), - None => continue, - }; - let task_def = match api.task_registry.get(&identifier) { - Some(t) => t, - None => continue, - }; - - let instance_ids: Vec = block.tasks.iter().map(|t| t.id.clone()).collect(); - let acquired_payloads: Vec>>> = block - .tasks - .iter() - .map(|t| Arc::new(std::sync::RwLock::new(t.task_payload.clone()))) - .collect(); - - Events::TaskBlockAcquired( - api.as_ref(), - task_id.clone(), - instance_ids.clone(), - acquired_payloads.clone(), - ); - - if Events::BeforeTaskBlockExecute( - api.as_ref(), - task_id.clone(), - instance_ids.clone(), - acquired_payloads.clone(), - ) { - continue; - } - - let task_id_for_execute = task_id.clone(); - let tasks_for_execute = block.tasks; - let payloads_for_execute = acquired_payloads.clone(); - let task_def_for_execute = task_def.clone(); - let execute_result = tokio::task::spawn_blocking(move || { - tasks_for_execute - .into_par_iter() - .zip(payloads_for_execute.into_par_iter()) - .filter_map(|(t, payload_lock)| { - let payload = match payload_lock.read() { - Ok(p) => p.clone(), - Err(_) => return None, - }; - let mut tsk = task_def_for_execute.clone().from_bytes(&payload); - tsk.run_hip(); - let out_bytes = tsk.to_bytes(); - let publish_payload_lock = - Arc::new(std::sync::RwLock::new(out_bytes.clone())); - let solved = proto::Task { - id: t.id, - task_id: task_id_for_execute.clone(), - task_payload: out_bytes, - payload: Vec::new(), - }; - Some((solved, publish_payload_lock)) - }) - .unzip::<_, _, Vec<_>, Vec<_>>() - }) - .await; - - let (mut solved, publish_payload_locks) = match execute_result { - Ok(result) => result, - Err(err) => { - debug!( - "worker {}: execute failed for {}: {:?}", - worker_id, task_id, err - ); - continue; - } - }; - - if solved.is_empty() { - continue; - } - - if Events::BeforeTaskBlockPublish( - api.as_ref(), - task_id.clone(), - instance_ids, - publish_payload_locks.clone(), - ) { - continue; - } - - // Sync any handler-modified payloads back into the outgoing block. - for (s, lock) in solved.iter_mut().zip(publish_payload_locks.iter()) { - if let Ok(p) = lock.read() { - s.task_payload = p.clone(); - } - } - - if let Err(status) = client - .publish_task_stream(Request::new(tokio_stream::iter(solved))) - .await - { - debug!( - "worker {}: publish failed for {}: {:?}", - worker_id, task_id, status - ); - } - } - - if !got_any { - tokio::time::sleep(std::time::Duration::from_millis(2)).await; - } - } -} - -// Compute Module -// Verifies server and also is -// Responsible for getting task, executing and publishing it. -async fn compute_module(api: Arc) { - let url = "http://[::1]:50051"; - let endpoint = Endpoint::from_static(url) - .tcp_nodelay(true) - .http2_adaptive_window(true) - .keep_alive_while_idle(true) - .tcp_keepalive(Some(std::time::Duration::from_secs(30))); - - let channel = endpoint.connect().await.unwrap(); - let interceptor = make_interceptor(api.clone()); - let mut client = engine_client::EngineClient::with_interceptor(channel.clone(), interceptor); - - // Get server metadata - let server_meta = client - .get_metadata(Request::new(proto::Empty {})) - .await - .unwrap() - .into_inner(); - - // validate server - assert!(server_meta.engine_api == enginelib::GIT_VERSION); - for x in &server_meta.mods { - assert!(x.api_version == enginelib::GIT_VERSION); - #[cfg(not(debug_assertions))] - assert!(x.rustc_version == enginelib::RUSTC_VERSION); - assert!(api.lib_manager.libraries.contains_key(&x.mod_id)); - let module: &LibraryInstance = api - .lib_manager - .libraries - .get(&x.mod_id) - .expect("Client Missing Mod"); - assert!(module.metadata.mod_version == x.mod_version) - } - - let task_reg = client - .aquire_task_reg(Request::new(proto::Empty {})) - .await - .unwrap() - .into_inner(); - - let task_ids = Arc::new(task_reg.tasks); - let worker_count = std::env::var("GE_CLIENT_WORKERS") - .ok() - .and_then(|v| v.parse::().ok()) - .filter(|v| *v > 0) - .unwrap_or_else(|| { - std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(4) - }); - let compute_worker_count = std::env::var("GE_CLIENT_COMPUTE_WORKERS") - .ok() - .and_then(|v| v.parse::().ok()) - .filter(|v| *v > 0) - .unwrap_or(worker_count); - let _ = rayon::ThreadPoolBuilder::new() - .num_threads(compute_worker_count) - .thread_name(|idx| format!("engine-client-cpu-{idx}")) - .build_global(); - - info!( - "Starting {} client workers and {} compute workers for {} task types", - worker_count, - compute_worker_count, - task_ids.len() - ); - - let mut handles = Vec::with_capacity(worker_count); - for worker_id in 0..worker_count { - let api_i = api.clone(); - let channel_i = channel.clone(); - let task_ids_i = task_ids.clone(); - handles.push(tokio::spawn(async move { - worker_loop(worker_id, api_i, channel_i, task_ids_i).await; - })); - } - - for handle in handles { - let _ = handle.await; - } -} diff --git a/engine/src/bin/packer.rs b/engine/src/bin/packer.rs deleted file mode 100644 index ba37461..0000000 --- a/engine/src/bin/packer.rs +++ /dev/null @@ -1,1068 +0,0 @@ -use clap::{Args, CommandFactory, Subcommand, ValueEnum, ValueHint}; -use clap::{Command, Parser}; -use clap_complete::{Generator, Shell, generate}; -use enginelib::Registry; -use enginelib::api::postcard; -use enginelib::events::{Events, ID}; -use enginelib::prelude::error; -use enginelib::task::StoredTask; -use enginelib::{api::ServerAPI, config::Config, event::info}; -use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; -use std::collections::HashMap; -use std::fs::File; -use std::io::Write; -use std::io::{self, BufReader, ErrorKind, Read}; -use std::path::PathBuf; -use toml::Value; -use tonic::{ - Request, - metadata::{MetadataKey, MetadataValue}, - transport::Endpoint, -}; - -pub mod proto { - tonic::include_proto!("engine"); -} - -#[derive(Debug)] -struct Entry { - namespace: String, - id: String, - data: BTreeMap, -} - -#[derive(Debug, Deserialize)] -#[serde(transparent)] -struct RawDoc( - std::collections::BTreeMap>>, -); - -fn parse_entries(raw: RawDoc) -> Vec { - let mut result = Vec::new(); - - for (compound_key, records) in raw.0 { - // split on colon: "widget:button" -> ("widget", "button") - let mut parts = compound_key.splitn(2, ':'); - let namespace = parts.next().unwrap_or("").to_string(); - let id = parts.next().unwrap_or("").to_string(); - - for data in records { - result.push(Entry { - namespace: namespace.clone(), - id: id.clone(), - data, - }); - } - } - - result -} - -/// A simple CLI application -#[derive(Parser, Debug)] -#[command(name = "packer")] -#[command(version = "1.0")] -#[command(author = "GrandEngineering")] -#[command(about = "A simple CLI app to pack tasks")] -struct Cli { - #[command(subcommand)] - command: Option, - #[arg(long = "generate", value_enum)] - generator: Option, -} - -#[derive(Subcommand, Debug, PartialEq)] -enum Commands { - #[command()] - Pack(PackArgs), - #[command()] - Unpack(PackArgs), - #[command()] - Upload(PackArgs), - #[command()] - AdminCheck, - #[command()] - AdminList(ListArgs), - #[command()] - AdminExport(ExportArgs), - #[command()] - AdminDelete(DeleteArgs), - #[command()] - Schema, -} - -#[derive(Args, Debug, PartialEq)] -struct PackArgs { - #[arg(short, required = true, value_hint = ValueHint::FilePath)] - input: PathBuf, - #[arg(long)] - stream: bool, -} - -#[derive(Clone, Debug, ValueEnum, PartialEq)] -enum StateArg { - Queued, - Processing, - Solved, -} - -impl StateArg { - fn to_proto(&self) -> i32 { - match self { - StateArg::Queued => proto::TaskState::Queued as i32, - StateArg::Processing => proto::TaskState::Leased as i32, - StateArg::Solved => proto::TaskState::Solved as i32, - } - } - - fn as_str(&self) -> &'static str { - match self { - StateArg::Queued => "queued", - StateArg::Processing => "processing", - StateArg::Solved => "solved", - } - } -} - -#[derive(Args, Debug, PartialEq)] -struct ListArgs { - #[arg(long)] - task_id: Option, // namespace:task - #[arg(long, value_enum, default_value = "queued")] - state: StateArg, - #[arg(long)] - all_states: bool, - #[arg(long, default_value_t = 1000)] - page_size: u32, -} - -#[derive(Args, Debug, PartialEq)] -struct ExportArgs { - #[arg(short = 'o', long, value_hint = ValueHint::FilePath, default_value = "output.rustforge.bin")] - output: PathBuf, - #[arg(long)] - task_id: Option, // namespace:task - #[arg(long, value_enum, default_value = "queued")] - state: StateArg, - #[arg(long)] - all_states: bool, - #[arg(long, default_value_t = 1000)] - page_size: u32, -} - -#[derive(Args, Debug, PartialEq)] -struct DeleteArgs { - #[arg(long)] - namespace: String, - #[arg(long)] - task: String, - #[arg(long)] - id: String, - #[arg(long, value_enum)] - state: StateArg, -} -fn print_completions(generator: G, cmd: &mut Command) { - generate( - generator, - cmd, - cmd.get_name().to_string(), - &mut io::stdout(), - ); -} - -fn build_headers(api: &ServerAPI, admin: bool) -> HashMap { - let headers = std::sync::Arc::new(std::sync::RwLock::new(HashMap::::new())); - Events::ClientAuthPrepare(api, headers.clone()); - let mut prepared_headers = headers.read().map(|h| h.clone()).unwrap_or_default(); - - if admin { - if let Some(token) = api.cfg.config_toml.cgrpc_token.clone() { - prepared_headers.insert("authorization".to_string(), token); - } - } - - prepared_headers -} - -const CHUNKED_MAGIC: [u8; 4] = *b"RFPK"; -const CHUNKED_VERSION: u8 = 2; -const FRAME_TASK: u8 = 1; -const FRAME_END: u8 = 255; - -#[derive(Debug)] -struct ChunkedTaskRecord { - namespace: String, - task: String, - id: String, - payload: Vec, -} - -#[derive(Debug, Default, Serialize, Deserialize)] -struct PackedTaskQueue { - tasks: BTreeMap<(String, String), Vec>, -} - -impl PackedTaskQueue { - fn push(&mut self, key: (String, String), task: StoredTask) { - self.tasks.entry(key).or_default().push(task); - } -} - -fn write_chunked_header(writer: &mut W) -> io::Result<()> { - writer.write_all(&CHUNKED_MAGIC)?; - writer.write_all(&[CHUNKED_VERSION])?; - Ok(()) -} - -fn write_chunked_task_record( - writer: &mut W, - record: &ChunkedTaskRecord, -) -> io::Result<()> { - let ns = record.namespace.as_bytes(); - let task = record.task.as_bytes(); - let id = record.id.as_bytes(); - - let ns_len: u16 = ns - .len() - .try_into() - .map_err(|_| io::Error::new(ErrorKind::InvalidInput, "namespace too large"))?; - let task_len: u16 = task - .len() - .try_into() - .map_err(|_| io::Error::new(ErrorKind::InvalidInput, "task too large"))?; - let id_len: u32 = id - .len() - .try_into() - .map_err(|_| io::Error::new(ErrorKind::InvalidInput, "id too large"))?; - let payload_len: u64 = record - .payload - .len() - .try_into() - .map_err(|_| io::Error::new(ErrorKind::InvalidInput, "payload too large"))?; - - let mut meta = [0u8; 1 + 2 + 2 + 4 + 8]; - meta[0] = FRAME_TASK; - meta[1..3].copy_from_slice(&ns_len.to_le_bytes()); - meta[3..5].copy_from_slice(&task_len.to_le_bytes()); - meta[5..9].copy_from_slice(&id_len.to_le_bytes()); - meta[9..17].copy_from_slice(&payload_len.to_le_bytes()); - - writer.write_all(&meta)?; - writer.write_all(ns)?; - writer.write_all(task)?; - writer.write_all(id)?; - writer.write_all(&record.payload)?; - - Ok(()) -} - -fn write_chunked_end(writer: &mut W) -> io::Result<()> { - writer.write_all(&[FRAME_END])?; - Ok(()) -} - -fn read_exact_or_eof(reader: &mut R, buf: &mut [u8]) -> io::Result { - match reader.read_exact(buf) { - Ok(()) => Ok(true), - Err(e) if e.kind() == ErrorKind::UnexpectedEof => Ok(false), - Err(e) => Err(e), - } -} - -fn read_chunked_header(reader: &mut R) -> io::Result<()> { - let mut magic = [0u8; 4]; - reader.read_exact(&mut magic)?; - if magic != CHUNKED_MAGIC { - return Err(io::Error::new( - ErrorKind::InvalidData, - "invalid chunked magic", - )); - } - - let mut version = [0u8; 1]; - reader.read_exact(&mut version)?; - if version[0] != CHUNKED_VERSION { - return Err(io::Error::new( - ErrorKind::InvalidData, - format!("unsupported chunked version {}", version[0]), - )); - } - - Ok(()) -} - -fn read_chunked_next(reader: &mut R) -> io::Result> { - let mut frame = [0u8; 1]; - if !read_exact_or_eof(reader, &mut frame)? { - return Ok(None); - } - - match frame[0] { - FRAME_END => Ok(None), - FRAME_TASK => { - let mut ns_len = [0u8; 2]; - let mut task_len = [0u8; 2]; - let mut id_len = [0u8; 4]; - let mut payload_len = [0u8; 8]; - reader.read_exact(&mut ns_len)?; - reader.read_exact(&mut task_len)?; - reader.read_exact(&mut id_len)?; - reader.read_exact(&mut payload_len)?; - - let ns_len = u16::from_le_bytes(ns_len) as usize; - let task_len = u16::from_le_bytes(task_len) as usize; - let id_len = u32::from_le_bytes(id_len) as usize; - let payload_len_u64 = u64::from_le_bytes(payload_len); - if payload_len_u64 > usize::MAX as u64 { - return Err(io::Error::new( - ErrorKind::InvalidData, - "payload length too large for this platform", - )); - } - let payload_len = payload_len_u64 as usize; - - let mut ns = vec![0u8; ns_len]; - let mut task = vec![0u8; task_len]; - let mut id = vec![0u8; id_len]; - let mut payload = vec![0u8; payload_len]; - - reader.read_exact(&mut ns)?; - reader.read_exact(&mut task)?; - reader.read_exact(&mut id)?; - reader.read_exact(&mut payload)?; - - let namespace = String::from_utf8(ns) - .map_err(|_| io::Error::new(ErrorKind::InvalidData, "invalid namespace utf8"))?; - let task = String::from_utf8(task) - .map_err(|_| io::Error::new(ErrorKind::InvalidData, "invalid task utf8"))?; - let id = String::from_utf8(id) - .map_err(|_| io::Error::new(ErrorKind::InvalidData, "invalid id utf8"))?; - - Ok(Some(ChunkedTaskRecord { - namespace, - task, - id, - payload, - })) - } - other => Err(io::Error::new( - ErrorKind::InvalidData, - format!("unknown frame type {}", other), - )), - } -} - -#[tokio::main] -async fn main() { - let cli = Cli::parse(); - if let Some(generator) = cli.generator { - let mut cmd = Cli::command(); - eprintln!("Generating completion file for {generator:?}..."); - print_completions(generator, &mut cmd); - } - let mut api = ServerAPI::default_client(); - ServerAPI::init_client(&mut api); - // packer intentionally uses client init path; load config explicitly for host/admin token - api.cfg = Config::new(); - if let Some(command) = cli.command { - match command { - Commands::Schema => { - let mut buf: Vec = Vec::new(); - for tsk in api.task_registry.tasks { - let unw = tsk.1.to_toml(); - buf.push(format![r#"[["{}:{}"]]"#, tsk.0.0, tsk.0.1]); - buf.push(unw); - } - let ns = buf.join("\n"); - match File::create("schema.rustforge.toml") { - Ok(mut file) => { - if let Err(e) = file.write_all(ns.as_bytes()) { - error!("Failed to write schema file: {}", e); - } else { - info!("Wrote schema.rustforge.toml"); - } - } - Err(e) => { - error!("Failed to create schema file: {}", e); - } - } - } - Commands::Unpack(input) => { - if input.input.exists() { - info!("Unpacking File: {}", input.input.to_string_lossy()); - let mut buf = Vec::new(); - - // Attempt to open and read the input file. If either step fails, - // we do not proceed to deserialization or writing the output file. - match File::open(&input.input) { - Ok(mut f) => { - if let Err(e) = f.read_to_end(&mut buf) { - error!( - "Failed to read input file {}: {}", - input.input.display(), - e - ); - // reading failed -> do not proceed to deserialize or write - return; - } - } - Err(e) => { - error!("Failed to open input file {}: {}", input.input.display(), e); - // opening failed -> do not proceed to deserialize or write - return; - } - } - - // Try to deserialize. Only on successful deserialization do we - // process entries and write the output TOML file. - let maybe_queue: Option = - match postcard::from_bytes::(&buf) { - Ok(k) => Some(k), - Err(e) => { - error!("Failed to deserialize task queue: {}", e); - None - } - }; - - if let Some(k) = maybe_queue { - let mut final_out: Vec = Vec::new(); - - for tasks in k.tasks { - match api.task_registry.tasks.get(&tasks.0.clone()) { - Some(tt) => { - for task in tasks.1 { - if tt.verify(task.bytes.clone()) { - let tmp_nt = tt.from_bytes(&task.bytes); - final_out.push(format![ - r#"[["{}:{}"]]"#, - tasks.0.0.clone(), - tasks.0.1.clone() - ]); - final_out.push(tmp_nt.to_toml()); - info!("{:?}", tmp_nt); - } - } - } - None => { - error!("Unknown template for {}:{}", tasks.0.0, tasks.0.1); - } - } - } - - let ns = final_out.join("\n"); - match File::create("output.rustforge.toml") { - Ok(mut file) => { - if let Err(e) = file.write_all(ns.as_bytes()) { - error!("Failed to write output.rustforge.toml: {}", e); - } else { - info!("Wrote output.rustforge.toml"); - } - } - Err(e) => { - error!("Failed to create output.rustforge.toml: {}", e); - } - } - } else { - // Deserialization failed; we logged the error above and intentionally do not - // create/write the output file to avoid producing an empty output. - } - } - } - Commands::Upload(input) => { - if !input.input.exists() { - error!("File does not exist: {}", input.input.to_string_lossy()); - return; - } - - info!( - "Uploading File: {}{}", - input.input.to_string_lossy(), - if input.stream { " (stream mode)" } else { "" } - ); - - let prepared_headers = build_headers(&api, false); - - let interceptor = move |mut req: Request<()>| { - for (key, value) in prepared_headers.iter() { - if let (Ok(key), Ok(value)) = ( - MetadataKey::from_bytes(key.as_bytes()), - MetadataValue::try_from(value.as_str()), - ) { - req.metadata_mut().insert(key, value); - } - } - Ok(req) - }; - - let endpoint = format!("http://{}", api.cfg.config_toml.host); - let channel = match Endpoint::from_shared(endpoint.clone()) { - Ok(ep) => match ep.connect().await { - Ok(ch) => ch, - Err(e) => { - error!("Failed to connect to server {}: {}", endpoint, e); - return; - } - }, - Err(e) => { - error!("Invalid server endpoint {}: {}", endpoint, e); - return; - } - }; - - let mut client = - proto::engine_client::EngineClient::with_interceptor(channel, interceptor); - - let mut uploaded = 0usize; - let mut failed = 0usize; - - if input.stream { - let file = match File::open(&input.input) { - Ok(f) => f, - Err(e) => { - error!("Failed to open input file {}: {}", input.input.display(), e); - return; - } - }; - let mut reader = BufReader::new(file); - if let Err(e) = read_chunked_header(&mut reader) { - error!("Failed to read chunked stream header: {}", e); - return; - } - - loop { - let record = match read_chunked_next(&mut reader) { - Ok(v) => v, - Err(e) => { - error!("Failed to read chunked stream record: {}", e); - return; - } - }; - - let Some(record) = record else { - break; - }; - - let task_id = format!("{}:{}", record.namespace, record.task); - let req = proto::Task { - id: record.id, - task_id: task_id.clone(), - task_payload: record.payload, - payload: Vec::new(), - }; - - match client - .create_task_block(Request::new(proto::TaskBlock { tasks: vec![req] })) - .await - { - Ok(resp) => uploaded += resp.into_inner().tasks.len(), - Err(e) => { - failed += 1; - error!("Failed to upload task {}: {}", task_id, e); - } - } - } - } else { - let mut buf = Vec::new(); - match File::open(&input.input) { - Ok(mut f) => { - if let Err(e) = f.read_to_end(&mut buf) { - error!( - "Failed to read input file {}: {}", - input.input.display(), - e - ); - return; - } - } - Err(e) => { - error!("Failed to open input file {}: {}", input.input.display(), e); - return; - } - } - - let queue: PackedTaskQueue = match postcard::from_bytes::(&buf) - { - Ok(q) => q, - Err(e) => { - error!("Failed to deserialize task queue: {}", e); - return; - } - }; - - for ((namespace, task), tasks) in queue.tasks { - let task_id = format!("{}:{}", namespace, task); - let requested = tasks.len(); - let requests: Vec = tasks - .into_iter() - .map(|stored| proto::Task { - id: stored.id, - task_id: task_id.clone(), - task_payload: stored.bytes, - payload: Vec::new(), - }) - .collect(); - - match client - .create_task_block(Request::new(proto::TaskBlock { tasks: requests })) - .await - { - Ok(resp) => uploaded += resp.into_inner().tasks.len(), - Err(e) => { - failed += requested; - error!("Failed to upload task {}: {}", task_id, e); - } - } - } - } - - info!( - "Upload complete. uploaded={}, failed={}, mode={}", - uploaded, - failed, - if input.stream { "stream" } else { "legacy" } - ); - } - Commands::AdminCheck => { - let prepared_headers = build_headers(&api, true); - let interceptor = move |mut req: Request<()>| { - for (key, value) in prepared_headers.iter() { - if let (Ok(key), Ok(value)) = ( - MetadataKey::from_bytes(key.as_bytes()), - MetadataValue::try_from(value.as_str()), - ) { - req.metadata_mut().insert(key, value); - } - } - Ok(req) - }; - - let endpoint = format!("http://{}", api.cfg.config_toml.host); - let channel = match Endpoint::from_shared(endpoint.clone()) { - Ok(ep) => match ep.connect().await { - Ok(ch) => ch, - Err(e) => { - error!("Failed to connect to server {}: {}", endpoint, e); - return; - } - }, - Err(e) => { - error!("Invalid server endpoint {}: {}", endpoint, e); - return; - } - }; - - let mut client = - proto::engine_client::EngineClient::with_interceptor(channel, interceptor); - match client.check_auth(Request::new(proto::Empty {})).await { - Ok(_) => info!("Admin auth check: OK"), - Err(e) => error!("Admin auth check failed: {}", e), - } - } - Commands::AdminList(args) => { - let prepared_headers = build_headers(&api, true); - let interceptor = move |mut req: Request<()>| { - for (key, value) in prepared_headers.iter() { - if let (Ok(key), Ok(value)) = ( - MetadataKey::from_bytes(key.as_bytes()), - MetadataValue::try_from(value.as_str()), - ) { - req.metadata_mut().insert(key, value); - } - } - Ok(req) - }; - - let endpoint = format!("http://{}", api.cfg.config_toml.host); - let channel = match Endpoint::from_shared(endpoint.clone()) { - Ok(ep) => match ep.connect().await { - Ok(ch) => ch, - Err(e) => { - error!("Failed to connect to server {}: {}", endpoint, e); - return; - } - }, - Err(e) => { - error!("Invalid server endpoint {}: {}", endpoint, e); - return; - } - }; - - let mut client = - proto::engine_client::EngineClient::with_interceptor(channel, interceptor); - - let task_ids: Vec = if let Some(task_id) = args.task_id.clone() { - vec![task_id] - } else { - match client.aquire_task_reg(Request::new(proto::Empty {})).await { - Ok(res) => res.into_inner().tasks, - Err(e) => { - error!("Failed to fetch task registry: {}", e); - return; - } - } - }; - - let states: Vec = if args.all_states { - vec![StateArg::Queued, StateArg::Processing, StateArg::Solved] - } else { - vec![args.state.clone()] - }; - - let mut listed = 0usize; - println!("state\ttask\tid"); - for task_id in task_ids { - let Some((namespace, task)) = task_id.split_once(':') else { - error!("Invalid task id '{}' (expected namespace:task)", task_id); - continue; - }; - - for state in &states { - let mut page = 0u64; - loop { - let req = proto::TaskPageRequest { - namespace: namespace.to_string(), - task: task.to_string(), - page, - page_size: args.page_size, - state: state.to_proto(), - }; - - let resp = match client.get_tasks(Request::new(req)).await { - Ok(r) => r.into_inner(), - Err(e) => { - error!( - "GetTasks failed for {} state {:?}: {}", - task_id, state, e - ); - break; - } - }; - - if resp.tasks.is_empty() { - break; - } - - for t in resp.tasks { - println!("{}\t{}:{}\t{}", state.as_str(), namespace, task, t.id); - listed += 1; - } - - page += 1; - } - } - } - - info!("Listed {} task(s)", listed); - } - Commands::AdminExport(args) => { - let prepared_headers = build_headers(&api, true); - let interceptor = move |mut req: Request<()>| { - for (key, value) in prepared_headers.iter() { - if let (Ok(key), Ok(value)) = ( - MetadataKey::from_bytes(key.as_bytes()), - MetadataValue::try_from(value.as_str()), - ) { - req.metadata_mut().insert(key, value); - } - } - Ok(req) - }; - - let endpoint = format!("http://{}", api.cfg.config_toml.host); - let channel = match Endpoint::from_shared(endpoint.clone()) { - Ok(ep) => match ep.connect().await { - Ok(ch) => ch, - Err(e) => { - error!("Failed to connect to server {}: {}", endpoint, e); - return; - } - }, - Err(e) => { - error!("Invalid server endpoint {}: {}", endpoint, e); - return; - } - }; - - let mut client = - proto::engine_client::EngineClient::with_interceptor(channel, interceptor); - - let task_ids: Vec = if let Some(task_id) = args.task_id.clone() { - vec![task_id] - } else { - match client.aquire_task_reg(Request::new(proto::Empty {})).await { - Ok(res) => res.into_inner().tasks, - Err(e) => { - error!("Failed to fetch task registry: {}", e); - return; - } - } - }; - - let states: Vec = if args.all_states { - vec![StateArg::Queued, StateArg::Processing, StateArg::Solved] - } else { - vec![args.state.clone()] - }; - - let mut out_queue = PackedTaskQueue::default(); - let mut fetched = 0usize; - - for task_id in task_ids { - let Some((namespace, task)) = task_id.split_once(':') else { - error!("Invalid task id '{}' (expected namespace:task)", task_id); - continue; - }; - - for state in &states { - let mut page = 0u64; - loop { - let req = proto::TaskPageRequest { - namespace: namespace.to_string(), - task: task.to_string(), - page, - page_size: args.page_size, - state: state.to_proto(), - }; - - let resp = match client.get_tasks(Request::new(req)).await { - Ok(r) => r.into_inner(), - Err(e) => { - error!( - "GetTasks failed for {} state {:?}: {}", - task_id, state, e - ); - break; - } - }; - - if resp.tasks.is_empty() { - break; - } - - for t in resp.tasks { - out_queue.push( - ID(namespace, task), - StoredTask { - bytes: t.task_payload, - id: t.id, - }, - ); - fetched += 1; - } - - page += 1; - } - } - } - - match postcard::to_allocvec(&out_queue) { - Ok(data) => match File::create(&args.output) { - Ok(mut file) => { - if let Err(e) = file.write_all(&data) { - error!("Failed to write {}: {}", args.output.display(), e); - } else { - info!( - "Export complete. wrote {} task(s) to {}", - fetched, - args.output.display() - ); - } - } - Err(e) => { - error!("Failed to create {}: {}", args.output.display(), e); - } - }, - Err(e) => error!("Failed to serialize export: {}", e), - } - } - Commands::AdminDelete(args) => { - let prepared_headers = build_headers(&api, true); - let interceptor = move |mut req: Request<()>| { - for (key, value) in prepared_headers.iter() { - if let (Ok(key), Ok(value)) = ( - MetadataKey::from_bytes(key.as_bytes()), - MetadataValue::try_from(value.as_str()), - ) { - req.metadata_mut().insert(key, value); - } - } - Ok(req) - }; - - let endpoint = format!("http://{}", api.cfg.config_toml.host); - let channel = match Endpoint::from_shared(endpoint.clone()) { - Ok(ep) => match ep.connect().await { - Ok(ch) => ch, - Err(e) => { - error!("Failed to connect to server {}: {}", endpoint, e); - return; - } - }, - Err(e) => { - error!("Invalid server endpoint {}: {}", endpoint, e); - return; - } - }; - - let mut client = - proto::engine_client::EngineClient::with_interceptor(channel, interceptor); - - let req = proto::TaskSelector { - state: args.state.to_proto(), - namespace: args.namespace.clone(), - task: args.task.clone(), - id: args.id.clone(), - }; - - match client.delete_task_block(Request::new(req)).await { - Ok(_) => info!( - "Deleted task {} from {}:{} ({:?})", - args.id, args.namespace, args.task, args.state - ), - Err(e) => error!("DeleteTask failed: {}", e), - } - } - Commands::Pack(input) => { - if input.input.exists() { - info!("Packing File: {}", input.input.to_string_lossy()); - match std::fs::read_to_string(&input.input) { - Ok(toml_str) => { - match toml::from_str::(&toml_str) { - Ok(raw) => { - let entries = parse_entries(raw); - let mut out_queue = PackedTaskQueue::default(); - for entry in entries { - match api - .task_registry - .get(&ID(entry.namespace.as_str(), entry.id.as_str())) - { - Some(template) => { - match toml::to_string(&entry.data) { - Ok(toml_string) => { - let t = template.from_toml(toml_string); - let key = ID( - entry.namespace.as_str(), - entry.id.as_str(), - ); - out_queue.push( - key, - StoredTask { - id: "".into(), //ids are minted on the server - bytes: t.to_bytes(), - }, - ); - } - Err(e) => { - error!( - "Failed to convert entry data to TOML string: {}", - e - ); - } - } - } - None => { - error!( - "Template not found for {}:{}", - entry.namespace, entry.id - ); - } - } - } - if input.stream { - match File::create("output.rustforge.bin") { - Ok(mut file) => { - if let Err(e) = write_chunked_header(&mut file) { - error!( - "Failed to write chunked output header: {}", - e - ); - return; - } - - let mut wrote = 0usize; - for ((namespace, task), tasks) in &out_queue.tasks { - for stored in tasks { - let record = ChunkedTaskRecord { - namespace: namespace.clone(), - task: task.clone(), - id: stored.id.clone(), - payload: stored.bytes.clone(), - }; - if let Err(e) = write_chunked_task_record( - &mut file, &record, - ) { - error!( - "Failed to write chunked output record: {}", - e - ); - return; - } - wrote += 1; - } - } - - if let Err(e) = write_chunked_end(&mut file) { - error!( - "Failed to finalize chunked output: {}", - e - ); - return; - } - - info!( - "Wrote output.rustforge.bin (chunked stream, {} task(s))", - wrote - ); - } - Err(e) => { - error!( - "Failed to create output.rustforge.bin: {}", - e - ); - } - } - } else { - match postcard::to_allocvec(&out_queue) { - Ok(data) => { - match File::create("output.rustforge.bin") { - Ok(mut file) => { - if let Err(e) = file.write_all(&data) { - error!( - "Failed to write output.rustforge.bin: {}", - e - ); - } else { - info!("Wrote output.rustforge.bin"); - } - } - Err(e) => { - error!( - "Failed to create output.rustforge.bin: {}", - e - ); - } - } - } - Err(e) => { - error!("Failed to serialize task queue: {}", e); - } - } - } - } - Err(e) => { - error!("Failed to parse input TOML: {}", e); - } - } - } - Err(e) => { - error!("Failed to read input file {}: {}", input.input.display(), e); - } - } - } else { - error!("File does not exist: {}", input.input.to_string_lossy()) - } - } - } - } -} diff --git a/engine/src/bin/server.rs b/engine/src/bin/server.rs index 09003da..f328e4d 100644 --- a/engine/src/bin/server.rs +++ b/engine/src/bin/server.rs @@ -1,45 +1 @@ -use engine::{EngineService, proto}; -use enginelib::{api::ServerAPI, event::info, events::Events}; -use std::{ - net::{Ipv4Addr, SocketAddr, SocketAddrV4}, - sync::Arc, -}; -use tokio::sync::RwLock; -use tonic::transport::Server; - -#[tokio::main(flavor = "multi_thread", worker_threads = 8)] -async fn main() -> Result<(), Box> { - let mut api = ServerAPI::default(); - ServerAPI::init(&mut api); - Events::init_auth(&mut api); - Events::StartEvent(&mut api); - Events::ServerStart(&api); - let addr = api - .cfg - .config_toml - .host - .parse() - .unwrap_or(SocketAddr::V4(SocketAddrV4::new( - Ipv4Addr::new(127, 0, 0, 1), - 50051, - ))); - let apii = Arc::new(RwLock::new(api)); - ServerAPI::init_chron(apii.clone()); - let engine = EngineService::new(apii); - - info!("Engine listening on {}", addr); - - let reflection_service = tonic_reflection::server::Builder::configure() - .register_encoded_file_descriptor_set(proto::FILE_DESCRIPTOR_SET) - .build_v1() - .map_err(|e| Box::new(e) as Box)?; - - Server::builder() - .add_service(reflection_service) - .add_service(engine.into_server()) - .serve(addr) - .await - .map_err(|e| Box::new(e) as Box)?; - - Ok(()) -} +fn main() {} diff --git a/engine/src/lib.rs b/engine/src/lib.rs deleted file mode 100644 index f2cff4b..0000000 --- a/engine/src/lib.rs +++ /dev/null @@ -1,705 +0,0 @@ -use enginelib::{ - Identifier, RawIdentifier, Registry, - api::ServerAPI, - chrono::Utc, - event::{debug, info, warn}, - events::{Events, ID}, - task::{LeasedTask, StoredTask, StoredTaskBlock}, -}; -use std::{ - collections::{HashMap, HashSet}, - sync::Arc, -}; -use tokio::sync::{RwLock, mpsc}; -use tokio_stream::wrappers::ReceiverStream; -use tonic::{Request, Response, Status}; - -pub mod proto { - tonic::include_proto!("engine"); - pub const FILE_DESCRIPTOR_SET: &[u8] = tonic::include_file_descriptor_set!("engine_descriptor"); -} - -use proto::{ - ModuleInfo, TaskState, - engine_server::{Engine, EngineServer}, -}; - -pub use proto::engine_server::EngineServer as EngineGrpcServer; - -pub fn get_uid(req: &Request) -> String { - req.metadata() - .get("uid") - .and_then(|v| v.to_str().ok()) - .map(|s| s.to_string()) - .unwrap_or_default() -} - -pub fn get_auth(req: &Request) -> String { - req.metadata() - .get("authorization") - .and_then(|v| v.to_str().ok()) - .map(|s| s.to_string()) - .unwrap_or_default() -} - -#[allow(non_snake_case)] -pub struct EngineService { - pub EngineAPI: Arc>, -} - -impl EngineService { - pub fn new(api: Arc>) -> Self { - Self { EngineAPI: api } - } - - pub fn into_server(self) -> EngineServer { - EngineServer::new(self) - } - - async fn acquire_task_block_impl( - &self, - request: tonic::Request, - ) -> Result { - let challenge = get_auth(&request); - let task_id = request.get_ref().task_id.clone(); - let requested_block_size = request.get_ref().block_size; - let uid = get_uid(&request); - - { - let api = self.EngineAPI.read().await; - let db = api.db.clone(); - if !Events::CheckAuth(&api, uid.clone(), challenge, db) { - return Err(Status::permission_denied("Invalid authentication")); - }; - } - - let (namespace, task_name) = task_id.split_once(':').ok_or_else(|| { - Status::invalid_argument("Invalid task ID format, expected 'namespace:task'") - })?; - let key = ID(namespace, task_name); - - { - let api = self.EngineAPI.read().await; - if api.task_registry.get(&key).is_none() { - return Err(Status::invalid_argument("Task Does not Exist")); - } - if Events::ServerBeforeTaskAcquire(&api, uid.clone(), task_id.clone()) { - return Err(Status::aborted( - "Task acquire cancelled by server event handler", - )); - } - } - - let (receiver, lock_arc) = { - let api = self.EngineAPI.read().await; - let receiver = api - .task_queue - .tasks - .get(&key) - .map(|entry| entry.0.clone()) - .ok_or_else(|| Status::not_found("Unknown task type"))?; - let lock_arc = api.fill_locks.entry(key.clone()).or_default().clone(); - (receiver, lock_arc) - }; - - if receiver.is_empty() { - let _fill_guard = lock_arc.lock().await; - if receiver.is_empty() { - let api = self.EngineAPI.read().await; - ServerAPI::fill_queue(&api, key.clone(), requested_block_size); - } - } - - let block = receiver - .recv() - .await - .map_err(|_| Status::unavailable("Task queue closed"))?; - - { - let api = self.EngineAPI.read().await; - let mut entry = api.leased_tasks.tasks.entry(key.clone()).or_default(); - let now = Utc::now(); - let mut ids = Vec::with_capacity(block.tasks.len()); - for task in &block.tasks { - ids.push(task.id.clone()); - entry.push(LeasedTask { - stored_task: Arc::new(task.clone()), - user_id: uid.clone(), - given_at: now, - }); - } - drop(entry); - api.clear_active_ids(&key, &ids); - Events::ServerTaskBlockAcquired(&api, uid.clone(), task_id.clone(), ids); - } - - let tasks = block - .tasks - .into_iter() - .map(|t| proto::Task { - id: t.id, - task_id: task_id.clone(), - task_payload: t.bytes, - payload: Vec::new(), - }) - .collect(); - - Ok(proto::TaskBlock { tasks }) - } - - async fn publish_tasks_impl( - &self, - uid: String, - challenge: String, - tasks: Vec, - ) -> Result<(), Status> { - if tasks.is_empty() { - return Ok(()); - } - - let api = self.EngineAPI.read().await; - - { - let db = api.db.clone(); - if !Events::CheckAuth(&api, uid.clone(), challenge, db) { - return Err(Status::permission_denied("Invalid authentication")); - }; - } - - let mut groups: HashMap> = HashMap::new(); - for t in tasks { - let Some((ns, name)) = t.task_id.split_once(':') else { - info!("publish: skipping malformed task_id {}", t.task_id); - continue; - }; - groups - .entry((ns.to_string(), name.to_string())) - .or_default() - .push(t); - } - - for (key, tasks) in groups { - let task_id_str = format!("{}:{}", key.0, key.1); - let Some(reg_tsk) = api.task_registry.get(&key) else { - info!("publish: unknown task {}:{}, skipping group", key.0, key.1); - continue; - }; - - let mut candidates: Vec<(proto::Task, Vec)> = Vec::with_capacity(tasks.len()); - let mut candidate_ids: HashSet = HashSet::with_capacity(tasks.len()); - let mut published_ids: Vec = Vec::with_capacity(tasks.len()); - - for t in tasks { - let payload_for_event = Arc::new(std::sync::RwLock::new(t.task_payload.clone())); - if Events::ServerBeforeTaskPublish( - &api, - uid.clone(), - task_id_str.clone(), - t.id.clone(), - payload_for_event.clone(), - ) { - info!("publish: handler cancelled {}:{}", task_id_str, t.id); - continue; - } - - let payload = match payload_for_event.read() { - Ok(p) => p.clone(), - Err(_) => { - info!("publish: payload lock poisoned for {}", t.id); - continue; - } - }; - - if !reg_tsk.clone().verify(payload.clone()) { - info!("publish: verify failed for {}", t.id); - continue; - } - - candidate_ids.insert(t.id.clone()); - candidates.push((t, payload)); - } - - let mut removed_ids: HashSet = HashSet::with_capacity(candidates.len()); - { - let mut leased = api.leased_tasks.tasks.entry(key.clone()).or_default(); - leased.retain(|lease| { - if lease.user_id == uid - && candidate_ids.contains(&lease.stored_task.id) - && !removed_ids.contains(&lease.stored_task.id) - { - removed_ids.insert(lease.stored_task.id.clone()); - false - } else { - true - } - }); - } - - for (t, payload) in candidates { - if !removed_ids.remove(&t.id) { - info!("publish: no lease for {} held by {}", t.id, uid); - continue; - } - let stored = StoredTask { - id: t.id.clone(), - bytes: payload, - }; - if let Err(e) = api.put_solved(&key, &stored) { - info!("publish: sled put_solved failed for {}: {}", t.id, e); - continue; - } - if let Err(e) = api.delete_queued(&key, &t.id) { - info!("publish: sled delete_queued failed for {}: {}", t.id, e); - } - published_ids.push(t.id); - } - - if !published_ids.is_empty() { - Events::ServerTaskBlockPublished(&api, uid.clone(), task_id_str, published_ids); - } - } - - Ok(()) - } -} - -#[tonic::async_trait] -impl Engine for EngineService { - async fn get_metadata( - &self, - _request: tonic::Request, - ) -> Result, Status> { - let api = self.EngineAPI.read().await; - - let modules: Vec = api - .lib_manager - .libraries - .values() - .map(|lib| lib.metadata.clone()) - .filter(|lib| !lib.mod_server) - .map(|f| ModuleInfo { - mod_id: f.mod_id.clone(), - api_version: f.api_version.clone(), - rustc_version: f.rustc_version.clone(), - mod_version: f.mod_version.clone(), - }) - .collect(); - - let res = proto::ServerMetadata { - engine_api: enginelib::GIT_VERSION.to_string(), - mods: modules, - }; - Ok(Response::new(res)) - } - - async fn check_auth( - &self, - request: tonic::Request, - ) -> Result, Status> { - let challenge = get_auth(&request); - let api = self.EngineAPI.read().await; - let db = api.db.clone(); - let output = Events::CheckAdminAuth(&api, challenge, ("".into(), "".into()), db); - if !output { - warn!("Auth check failed - permission denied"); - return Err(tonic::Status::permission_denied("Invalid Auth")); - }; - Ok(tonic::Response::new(proto::Empty {})) - } - - async fn delete_task_block( - &self, - request: tonic::Request, - ) -> Result, Status> { - let api = self.EngineAPI.read().await; - let data = request.get_ref(); - let challenge = get_auth(&request); - let db = api.db.clone(); - let key = ID(&data.namespace, &data.task); - - if !Events::CheckAdminAuth(&api, challenge, ("".into(), "".into()), db) { - warn!("Auth check failed - permission denied"); - return Err(tonic::Status::permission_denied("Invalid Auth")); - }; - - let removed = match data.state() { - TaskState::Queued => api - .delete_queued(&key, &data.id) - .map_err(|e| Status::internal(format!("sled: {e}")))?, - TaskState::Solved => api - .delete_solved(&key, &data.id) - .map_err(|e| Status::internal(format!("sled: {e}")))?, - TaskState::Leased => { - let mut leased = api.leased_tasks.tasks.entry(key.clone()).or_default(); - let orig = leased.len(); - leased.retain(|l| l.stored_task.id != data.id); - orig != leased.len() - } - }; - - if !removed { - return Err(Status::not_found(format!( - "Task {} not found in {:?} for {}:{}", - data.id, - data.state(), - data.namespace, - data.task, - ))); - } - - info!( - "DeleteTask: deleted {} in {:?} for {}:{}", - data.id, - data.state(), - data.namespace, - data.task - ); - Ok(tonic::Response::new(proto::Empty {})) - } - - async fn get_tasks( - &self, - request: tonic::Request, - ) -> std::result::Result, tonic::Status> { - let api = self.EngineAPI.read().await; - let challenge = get_auth(&request); - - let db = api.db.clone(); - if !Events::CheckAdminAuth(&api, challenge, ("".into(), "".into()), db) { - info!("GetTask denied due to Invalid Auth"); - return Err(Status::permission_denied("Invalid authentication")); - }; - let data = request.get_ref(); - let key = ID(&data.namespace, &data.task); - let task_id_str = format!("{}:{}", data.namespace, data.task); - - let to_proto = |id: String, bytes: Vec| proto::Task { - id, - task_id: task_id_str.clone(), - task_payload: bytes, - payload: Vec::new(), - }; - - let mut tasks: Vec = match data.state() { - TaskState::Queued => { - let mut v: Vec<_> = api - .scan_queued(&key) - .map(|t| to_proto(t.id, t.bytes)) - .collect(); - v.sort_by(|a, b| a.id.cmp(&b.id)); - v - } - TaskState::Solved => { - let mut v: Vec<_> = api - .scan_solved(&key) - .map(|t| to_proto(t.id, t.bytes)) - .collect(); - v.sort_by(|a, b| a.id.cmp(&b.id)); - v - } - TaskState::Leased => { - let mut v: Vec<_> = api - .leased_tasks - .tasks - .get(&key) - .map(|leased| { - leased - .iter() - .map(|l| { - to_proto(l.stored_task.id.clone(), l.stored_task.bytes.clone()) - }) - .collect() - }) - .unwrap_or_default(); - v.sort_by(|a, b| a.id.cmp(&b.id)); - v - } - }; - - let page_size = api.cfg.config_toml.pagination_limit.min(data.page_size) as usize; - let start = (data.page as usize).saturating_mul(page_size); - let end = start.saturating_add(page_size).min(tasks.len()); - let final_vec = if start >= tasks.len() { - Vec::new() - } else { - tasks.drain(start..end).collect() - }; - - Ok(tonic::Response::new(proto::TaskPage { - namespace: data.namespace.clone(), - task: data.task.clone(), - page: data.page, - page_size: page_size as u32, - state: data.state, - tasks: final_vec, - })) - } - - async fn cgrpc( - &self, - request: tonic::Request, - ) -> std::result::Result, tonic::Status> { - info!( - "CGRPC request received for handler: {}:{}", - request.get_ref().handler_mod_id, - request.get_ref().handler_id - ); - let mut api = self.EngineAPI.write().await; - let challenge = get_auth(&request); - let db = api.db.clone(); - debug!("Checking admin authentication for CGRPC request"); - let output = Events::CheckAdminAuth( - &mut api, - challenge, - ( - request.get_ref().handler_mod_id.clone(), - request.get_ref().handler_id.clone(), - ), - db, - ); - if !output { - warn!("CGRPC auth check failed - permission denied"); - return Err(tonic::Status::permission_denied("Invalid CGRPC Auth")); - }; - let out = Arc::new(std::sync::RwLock::new(Vec::new())); - debug!("Dispatching CGRPC event to handler"); - Events::CgrpcEvent( - &mut api, - ID("engine_core", "grpc"), - request.get_ref().event_payload.clone(), - out.clone(), - ); - let mut res = request.get_ref().clone(); - res.event_payload = match out.read() { - Ok(g) => g.clone(), - Err(_) => { - warn!("CGRPC response lock poisoned, returning empty payload"); - Vec::new() - } - }; - info!("CGRPC request processed successfully"); - Ok(tonic::Response::new(res)) - } - - async fn aquire_task_reg( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let uid = get_uid(&request); - let challenge = get_auth(&request); - info!("Task registry request received from user: {}", uid); - let api = self.EngineAPI.read().await; - let db = api.db.clone(); - - debug!("Validating authentication for task registry request"); - if !Events::CheckAuth(&api, uid.clone(), challenge, db) { - info!( - "Task registry request denied - invalid authentication for user: {}", - uid - ); - return Err(Status::permission_denied("Invalid authentication")); - }; - let mut tasks: Vec = Vec::new(); - for entry in api.task_registry.tasks.iter() { - let k = entry.key(); - tasks.push(format!("{}:{}", k.0, k.1)); - } - info!("Returning task registry with {} tasks", tasks.len()); - let response = proto::TaskRegistry { tasks }; - Ok(tonic::Response::new(response)) - } - - async fn aquire_task_block( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - self.acquire_task_block_impl(request) - .await - .map(Response::new) - } - - type AquireTaskStreamStream = ReceiverStream>; - - async fn aquire_task_stream( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let block = self.acquire_task_block_impl(request).await?; - let (tx, rx) = mpsc::channel(block.tasks.len().max(1)); - - for task in block.tasks { - if tx.send(Ok(task)).await.is_err() { - break; - } - } - - Ok(Response::new(ReceiverStream::new(rx))) - } - - async fn publish_task_block( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let challenge = get_auth(&request); - let uid = get_uid(&request); - self.publish_tasks_impl(uid, challenge, request.into_inner().tasks) - .await?; - Ok(tonic::Response::new(proto::Empty {})) - } - - async fn publish_task_stream( - &self, - request: tonic::Request>, - ) -> Result, tonic::Status> { - let challenge = get_auth(&request); - let uid = get_uid(&request); - let batch_size = { - let api = self.EngineAPI.read().await; - api.cfg.config_toml.task_block_size.max(1) as usize - }; - let mut stream = request.into_inner(); - let mut batch = Vec::with_capacity(batch_size.min(1024)); - - while let Some(task) = stream.message().await? { - batch.push(task); - if batch.len() >= batch_size { - self.publish_tasks_impl(uid.clone(), challenge.clone(), std::mem::take(&mut batch)) - .await?; - } - } - - if !batch.is_empty() { - self.publish_tasks_impl(uid, challenge, batch).await?; - } - - Ok(tonic::Response::new(proto::Empty {})) - } - - async fn create_task_block( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let challenge = get_auth(&request); - let uid = get_uid(&request); - let api = self.EngineAPI.read().await; - let db = api.db.clone(); - if !Events::CheckAuth(&api, uid, challenge, db) { - info!("Create Task denied due to Invalid Auth"); - return Err(Status::permission_denied("Invalid authentication")); - }; - - let mut groups: HashMap> = HashMap::new(); - for t in request.into_inner().tasks { - let Some((ns, name)) = t.task_id.split_once(':') else { - return Err(Status::invalid_argument(format!( - "Invalid task ID format: {}", - t.task_id - ))); - }; - if ns.is_empty() || name.is_empty() { - return Err(Status::invalid_argument( - "Invalid task ID format, expected 'namespace:task'", - )); - } - groups - .entry((ns.to_string(), name.to_string())) - .or_default() - .push(t); - } - - let mut created: Vec = Vec::new(); - - for (key, tasks) in groups { - let task_id_str = format!("{}:{}", key.0, key.1); - let Some(reg_tsk) = api.task_registry.get(&key) else { - return Err(Status::invalid_argument(format!( - "Task does not exist: {}", - task_id_str - ))); - }; - let enqueue_to_memory = api - .task_queue - .tasks - .get(&key) - .map(|channel| { - channel.1.len() < api.cfg.config_toml.task_queue_size.max(1) as usize - }) - .unwrap_or(false); - - let mut block_tasks: Vec = Vec::with_capacity(tasks.len()); - let mut instance_ids: Vec = Vec::with_capacity(tasks.len()); - let mut active_ids: Vec = Vec::with_capacity(tasks.len()); - let mut payloads: Vec>>> = - Vec::with_capacity(tasks.len()); - let mut created_for_group: Vec = Vec::with_capacity(tasks.len()); - let mut stored_for_group: Vec = Vec::with_capacity(tasks.len()); - - for t in tasks { - let payload_for_event = Arc::new(std::sync::RwLock::new(t.task_payload.clone())); - if Events::ServerBeforeTaskCreate( - &api, - task_id_str.clone(), - payload_for_event.clone(), - ) { - info!("create: handler cancelled task in {}", task_id_str); - continue; - } - let payload = match payload_for_event.read() { - Ok(p) => p.clone(), - Err(_) => { - info!("create: payload lock poisoned in {}", task_id_str); - continue; - } - }; - if !reg_tsk.clone().verify(payload.clone()) { - info!("create: verify failed in {}", task_id_str); - continue; - } - let stored = StoredTask { - id: druid::Druid::default().to_hex(), - bytes: payload, - }; - if enqueue_to_memory { - api.mark_active_id(&key, stored.id.clone()); - active_ids.push(stored.id.clone()); - } - instance_ids.push(stored.id.clone()); - payloads.push(payload_for_event); - created_for_group.push(proto::Task { - id: stored.id.clone(), - task_id: task_id_str.clone(), - task_payload: stored.bytes.clone(), - payload: Vec::new(), - }); - if enqueue_to_memory { - block_tasks.push(stored.clone()); - } - stored_for_group.push(stored); - } - - if !instance_ids.is_empty() { - if let Err(e) = api.put_queued_batch(&key, &stored_for_group) { - api.clear_active_ids(&key, &active_ids); - info!("create: sled batch failed for {}: {}", task_id_str, e); - continue; - } - - created.extend(created_for_group); - if let Some(channel) = api.task_queue.tasks.get(&key) { - if enqueue_to_memory - && !block_tasks.is_empty() - && channel - .1 - .try_send(StoredTaskBlock { tasks: block_tasks }) - .is_err() - { - api.clear_active_ids(&key, &active_ids); - } - } - Events::ServerTaskBlockCreated(&api, task_id_str, instance_ids, payloads); - } - } - - Ok(tonic::Response::new(proto::TaskBlock { tasks: created })) - } -} diff --git a/engine/tests/task_streaming_tests.rs b/engine/tests/task_streaming_tests.rs deleted file mode 100644 index 9e8c179..0000000 --- a/engine/tests/task_streaming_tests.rs +++ /dev/null @@ -1,180 +0,0 @@ -use engine::{EngineService, proto}; -use enginelib::{ - Identifier, Registry, - api::ServerAPI, - task::{StoredTask, StoredTaskBlock, Task, Verifiable}, -}; -use std::{ - net::{Ipv4Addr, SocketAddr, SocketAddrV4}, - sync::Arc, - time::{Duration, Instant}, -}; -use tokio::{net::TcpListener, sync::RwLock}; -use tonic::{Request, transport::Server}; - -const TEST_NS: &str = "stream_test"; -const TEST_TASK: &str = "noop"; - -#[derive(Debug, Clone, Default)] -struct NoopTask; - -impl Verifiable for NoopTask { - fn verify(&self, _b: Vec) -> bool { - true - } -} - -impl Task for NoopTask { - fn get_id(&self) -> Identifier { - (TEST_NS.to_string(), TEST_TASK.to_string()) - } - - fn clone_box(&self) -> Box { - Box::new(self.clone()) - } - - fn run_cpu(&mut self) {} - - fn to_bytes(&self) -> Vec { - Vec::new() - } - - fn from_bytes(&self, _bytes: &[u8]) -> Box { - Box::new(self.clone()) - } - - fn from_toml(&self, _d: String) -> Box { - Box::new(self.clone()) - } - - fn to_toml(&self) -> String { - String::new() - } -} - -async fn pick_port() -> u16 { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - listener.local_addr().unwrap().port() -} - -async fn build_client(port: u16) -> proto::engine_client::EngineClient { - let url = format!("http://127.0.0.1:{port}"); - let deadline = Instant::now() + Duration::from_secs(5); - - loop { - let endpoint = tonic::transport::Endpoint::try_from(url.clone()) - .unwrap() - .tcp_nodelay(true); - match endpoint.connect().await { - Ok(channel) => return proto::engine_client::EngineClient::new(channel), - Err(err) if Instant::now() < deadline => { - tokio::time::sleep(Duration::from_millis(20)).await; - let _ = err; - } - Err(err) => panic!("failed to connect to test server on {url}: {err}"), - } - } -} - -async fn spawn_test_server(port: u16) -> Arc> { - let mut api = ServerAPI::test_default(); - let id = (TEST_NS.to_string(), TEST_TASK.to_string()); - api.task_registry.register(Arc::new(NoopTask), id.clone()); - api.ensure_task_channel(id); - enginelib::event::register_inventory_handlers(&mut api); - - let api = Arc::new(RwLock::new(api)); - let engine = EngineService::new(api.clone()); - let addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, port)); - - tokio::spawn(async move { - let _ = Server::builder() - .add_service(engine.into_server()) - .serve(addr) - .await; - }); - - api -} - -#[tokio::test(flavor = "multi_thread")] -async fn acquire_and_publish_task_streams() { - let port = pick_port().await; - let api = spawn_test_server(port).await; - let mut client = build_client(port).await; - let task_id = format!("{TEST_NS}:{TEST_TASK}"); - - let created = client - .create_task_block(Request::new(proto::TaskBlock { - tasks: (0..3) - .map(|i| proto::Task { - id: String::new(), - task_id: task_id.clone(), - task_payload: vec![i], - payload: Vec::new(), - }) - .collect(), - })) - .await - .unwrap() - .into_inner(); - assert_eq!(created.tasks.len(), 3); - - let mut stream = client - .aquire_task_stream(Request::new(proto::TaskBlockRequest { - task_id: task_id.clone(), - block_size: 3, - })) - .await - .unwrap() - .into_inner(); - - let mut acquired = Vec::new(); - while let Some(task) = stream.message().await.unwrap() { - acquired.push(task); - } - assert_eq!(acquired.len(), 3); - - client - .publish_task_stream(Request::new(tokio_stream::iter(acquired))) - .await - .unwrap(); - - let key = (TEST_NS.to_string(), TEST_TASK.to_string()); - let api = api.read().await; - assert_eq!(api.scan_solved(&key).count(), 3); - assert_eq!(api.scan_queued(&key).count(), 0); -} - -#[test] -fn refill_skips_active_tasks_between_recv_and_lease() { - let mut api = ServerAPI::test_default(); - let key = (TEST_NS.to_string(), TEST_TASK.to_string()); - api.task_registry.register(Arc::new(NoopTask), key.clone()); - api.ensure_task_channel(key.clone()); - - let stored = StoredTask { - id: "active-task".to_string(), - bytes: vec![1], - }; - api.mark_active_id(&key, stored.id.clone()); - api.put_queued(&key, &stored).unwrap(); - - let (receiver, sender) = { - let channel = api.task_queue.tasks.get(&key).unwrap(); - (channel.0.clone(), channel.1.clone()) - }; - sender - .try_send(StoredTaskBlock { - tasks: vec![stored], - }) - .unwrap(); - - let received = receiver.try_recv().unwrap(); - assert_eq!(received.tasks.len(), 1); - assert!(receiver.is_empty()); - - ServerAPI::fill_queue(&api, key, 1); - - assert!(receiver.try_recv().is_err()); -} diff --git a/enginelib/Cargo.toml b/enginelib/Cargo.toml index 241f162..f1bbe62 100644 --- a/enginelib/Cargo.toml +++ b/enginelib/Cargo.toml @@ -25,9 +25,7 @@ dashmap = "6.2.1" async-channel = "2.5.0" [build-dependencies] vergen-gix = { version = "9.1.0", features = ["build", "cargo", "rustc"] } -[profile.release] -codegen-units = 1 # Make builds deterministic -[profile.dev] -codegen-units = 1 # Make builds deterministic +# Profiles moved to the workspace root Cargo.toml — Cargo ignores profile +# tables in non-root member manifests. [dev-dependencies] tracing-test = "0.2.6" diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index 3dc5510..1ad6376 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -1,8 +1,7 @@ use chrono::Utc; -use crossbeam::queue::ArrayQueue; -use dashmap::DashMap; -use tokio::{spawn, sync::RwLock, time::interval}; -use tracing::{Level, debug, error, info, instrument}; +use dashmap::{DashMap, DashSet}; +use tokio::{spawn, time::interval}; +use tracing::{Level, debug, info, instrument}; use crate::{ Identifier, Registry, @@ -15,397 +14,16 @@ pub use postcard; pub use postcard::from_bytes; pub use postcard::to_allocvec; use std::{ - collections::{HashMap, HashSet}, - sync::Arc, - time::Duration, + collections::HashMap, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + time::{Duration, SystemTime, UNIX_EPOCH}, }; pub struct ServerAPI { - pub cfg: Config, // RW - pub task_queue: TaskQueue, // RW - pub leased_tasks: LeasedTaskQueue, // RW - pub active_task_ids: DashMap>, - pub task_registry: EngineTaskRegistry, // RW - pub event_bus: EventBus, // RW - pub db: sled::Db, // R - pub lib_manager: LibraryManager, // RW - // Serializes only sled refills per Identifier. Active task ids keep refills - // from duplicating tasks already queued in memory or between recv+lease. - pub fill_locks: DashMap>>, -} - -impl Default for ServerAPI { - fn default() -> Self { - Self { - cfg: Config::default(), - task_queue: TaskQueue::default(), - db: sled::open("engine_db").unwrap(), - lib_manager: LibraryManager::default(), - task_registry: EngineTaskRegistry::default(), - event_bus: EventBus { - event_handler_registry: EngineEventHandlerRegistry { - event_handlers: HashMap::new(), - }, - }, - leased_tasks: LeasedTaskQueue::default(), - active_task_ids: DashMap::new(), - fill_locks: DashMap::new(), - } - } -} -impl ServerAPI { - fn temporary_db_path(prefix: &str) -> std::path::PathBuf { - use std::sync::atomic::{AtomicUsize, Ordering}; - - static DB_COUNTER: AtomicUsize = AtomicUsize::new(0); - let db_id = DB_COUNTER.fetch_add(1, Ordering::Relaxed); - let root = std::env::var_os("ENGINE_TEST_DB_ROOT") - .map(std::path::PathBuf::from) - .unwrap_or_else(|| { - std::env::current_dir() - .unwrap_or_else(|_| std::env::temp_dir()) - .join("target") - .join("tmp") - }); - let _ = std::fs::create_dir_all(&root); - root.join(format!("{}-{}-{}", prefix, std::process::id(), db_id)) - } - - pub fn test_default() -> Self { - let db_path = Self::temporary_db_path("enginelib-test-db"); - - Self { - cfg: Config::new(), - leased_tasks: LeasedTaskQueue::default(), - task_queue: TaskQueue::default(), - db: sled::Config::new() - .path(db_path) - .temporary(true) - .flush_every_ms(None) - .open() - .unwrap(), - lib_manager: LibraryManager::default(), - task_registry: EngineTaskRegistry::default(), - event_bus: EventBus { - event_handler_registry: EngineEventHandlerRegistry { - event_handlers: HashMap::new(), - }, - }, - fill_locks: DashMap::new(), - active_task_ids: DashMap::new(), - } - } - /// Ensure the task_queue + leased_tasks entries exist for this Identifier. - /// Idempotent — calling repeatedly is a no-op once registered. - pub fn ensure_task_channel(&self, id: Identifier) { - self.task_queue.tasks.entry(id.clone()).or_insert_with(|| { - let (s, r) = async_channel::unbounded(); - (r, s) - }); - self.leased_tasks.tasks.entry(id.clone()).or_default(); - self.active_task_ids.entry(id).or_default(); - } - - pub fn init(api: &mut Self) { - Self::setup_logger(); - api.cfg = Config::new(); - Self::init_db(api); - let mut new_lib_manager = LibraryManager::default(); - new_lib_manager.load_modules(api); - api.lib_manager = new_lib_manager; - for (id, _tsk) in api.task_registry.tasks.clone() { - let (s, r) = async_channel::unbounded(); - api.task_queue.tasks.entry(id.clone()).insert((r, s)); - api.leased_tasks.tasks.entry(id.clone()).or_default(); - api.active_task_ids.entry(id.clone()).or_default(); - } - - Self::init_events(api); - } - - /// Client-side ServerAPI with a temp sled (client doesn't use db/task_queue - /// at all; the temp dir is just to satisfy the struct field). - pub fn default_client() -> Self { - let db_path = Self::temporary_db_path("enginelib-client-db"); - Self { - cfg: Config::new(), - task_queue: TaskQueue::default(), - leased_tasks: LeasedTaskQueue::default(), - task_registry: EngineTaskRegistry::default(), - event_bus: EventBus { - event_handler_registry: EngineEventHandlerRegistry { - event_handlers: HashMap::new(), - }, - }, - db: sled::Config::new() - .path(db_path) - .temporary(true) - .flush_every_ms(None) - .open() - .unwrap(), - lib_manager: LibraryManager::default(), - fill_locks: DashMap::new(), - active_task_ids: DashMap::new(), - } - } - - /// Client init: logger + inventory event handlers. Skips load_modules - /// because client mods are loaded via a different path and module - /// validation happens against server metadata. - pub fn init_client(api: &mut Self) { - Self::setup_logger(); - Self::init_events(api); - } - - fn init_events(api: &mut Self) { - crate::event::register_inventory_handlers(api); - } - pub fn init_chron(api: Arc>) { - let t = api.try_read().unwrap().cfg.config_toml.clean_tasks; - spawn(clear_sled_periodically(api, t)); - } - // type:namespace:task:id - pub const TASKS_PREFIX: &'static str = "tasks:"; - pub const SOLVED_PREFIX: &'static str = "solved:"; - - fn state_key(prefix: &str, task_id: &Identifier, id: &str) -> Vec { - format!("{}{}\u{1f}{}:{}", prefix, task_id.0, task_id.1, id).into_bytes() - } - - fn state_prefix(prefix: &str, task_id: &Identifier) -> Vec { - format!("{}{}\u{1f}{}:", prefix, task_id.0, task_id.1).into_bytes() - } - - pub fn task_key(task_id: &Identifier, id: &str) -> Vec { - Self::state_key(Self::TASKS_PREFIX, task_id, id) - } - - pub fn solved_key(task_id: &Identifier, id: &str) -> Vec { - Self::state_key(Self::SOLVED_PREFIX, task_id, id) - } - - pub fn task_prefix(task_id: &Identifier) -> Vec { - Self::state_prefix(Self::TASKS_PREFIX, task_id) - } - - pub fn solved_prefix(task_id: &Identifier) -> Vec { - Self::state_prefix(Self::SOLVED_PREFIX, task_id) - } - - pub fn put_queued(&self, task_id: &Identifier, task: &StoredTask) -> sled::Result<()> { - let bytes = postcard::to_allocvec(task) - .map_err(|e| sled::Error::Unsupported(format!("postcard: {e}")))?; - self.db.insert(Self::task_key(task_id, &task.id), bytes)?; - Ok(()) - } - - pub fn put_queued_batch(&self, task_id: &Identifier, tasks: &[StoredTask]) -> sled::Result<()> { - let mut batch = sled::Batch::default(); - for task in tasks { - let bytes = postcard::to_allocvec(task) - .map_err(|e| sled::Error::Unsupported(format!("postcard: {e}")))?; - batch.insert(Self::task_key(task_id, &task.id), bytes); - } - self.db.apply_batch(batch) - } - - pub fn put_solved(&self, task_id: &Identifier, task: &StoredTask) -> sled::Result<()> { - let bytes = postcard::to_allocvec(task) - .map_err(|e| sled::Error::Unsupported(format!("postcard: {e}")))?; - self.db.insert(Self::solved_key(task_id, &task.id), bytes)?; - Ok(()) - } - - pub fn delete_queued(&self, task_id: &Identifier, id: &str) -> sled::Result { - Ok(self.db.remove(Self::task_key(task_id, id))?.is_some()) - } - - pub fn delete_solved(&self, task_id: &Identifier, id: &str) -> sled::Result { - Ok(self.db.remove(Self::solved_key(task_id, id))?.is_some()) - } - - pub fn scan_queued(&self, task_id: &Identifier) -> impl Iterator + '_ { - self.db - .scan_prefix(Self::task_prefix(task_id)) - .filter_map(|item| item.ok()) - .filter_map(|(_, value)| postcard::from_bytes::(&value).ok()) - } - - pub fn scan_solved(&self, task_id: &Identifier) -> impl Iterator + '_ { - self.db - .scan_prefix(Self::solved_prefix(task_id)) - .filter_map(|item| item.ok()) - .filter_map(|(_, value)| postcard::from_bytes::(&value).ok()) - } - - pub fn mark_active_id(&self, task_id: &Identifier, id: String) { - self.active_task_ids - .entry(task_id.clone()) - .or_default() - .insert(id); - } - - pub fn mark_active_ids(&self, task_id: &Identifier, ids: &[String]) { - if ids.is_empty() { - return; - } - - let mut active = self.active_task_ids.entry(task_id.clone()).or_default(); - for id in ids { - active.insert(id.clone()); - } - } - - pub fn clear_active_ids(&self, task_id: &Identifier, ids: &[String]) { - if ids.is_empty() { - return; - } - - if let Some(mut active) = self.active_task_ids.get_mut(task_id) { - for id in ids { - active.remove(id); - } - } - } - - pub fn fill_queue(api: &ServerAPI, task_id: Identifier, requested_block_size: u32) { - let max_block = if requested_block_size > 0 { - requested_block_size as usize - } else { - api.cfg.config_toml.task_block_size.max(1) as usize - }; - let max_queue = api.cfg.config_toml.task_queue_size as usize; - - let Some(channel) = api.task_queue.tasks.get(&task_id) else { - return; - }; - let sender = &channel.1; - - // Soft size lock: bail out if the channel is already at its configured cap. - // The cap is advisory — a partially-built trailing block may still push us - // one block past the limit, but no full block is enqueued once we hit it. - if sender.len() >= max_queue { - return; - } - - // Build a leased-id set once per call — O(L) instead of O(N·L) per scan item. - let leased: HashSet = api - .leased_tasks - .tasks - .get(&task_id) - .map(|v| v.iter().map(|l| l.stored_task.id.clone()).collect()) - .unwrap_or_default(); - let active: HashSet = api - .active_task_ids - .get(&task_id) - .map(|v| v.iter().cloned().collect()) - .unwrap_or_default(); - - let mut block: Vec = Vec::with_capacity(max_block); - - for item in api.db.scan_prefix(Self::task_prefix(&task_id)) { - let Ok((_, value)) = item else { continue }; - let Ok(task) = postcard::from_bytes::(&value) else { - continue; - }; - let is_active = active.contains(&task.id) - || api - .active_task_ids - .get(&task_id) - .map(|ids| ids.contains(&task.id)) - .unwrap_or(false); - if leased.contains(&task.id) || is_active { - continue; - } - - block.push(task); - if block.len() == max_block { - let full = std::mem::replace(&mut block, Vec::with_capacity(max_block)); - let ids: Vec = full.iter().map(|task| task.id.clone()).collect(); - api.mark_active_ids(&task_id, &ids); - if sender.try_send(StoredTaskBlock { tasks: full }).is_err() { - api.clear_active_ids(&task_id, &ids); - return; // receiver dropped - } - if sender.len() >= max_queue { - return; - } - } - } - - if !block.is_empty() { - let ids: Vec = block.iter().map(|task| task.id.clone()).collect(); - api.mark_active_ids(&task_id, &ids); - if sender.try_send(StoredTaskBlock { tasks: block }).is_err() { - api.clear_active_ids(&task_id, &ids); - } - } - } - - fn init_db(api: &mut ServerAPI) { - api.task_queue = TaskQueue::default(); - api.leased_tasks = LeasedTaskQueue::default(); - api.active_task_ids = DashMap::new(); - } - - pub fn setup_logger() { - use std::sync::OnceLock; - - static INIT: OnceLock<()> = OnceLock::new(); - INIT.get_or_init(|| { - #[cfg(debug_assertions)] - let _ = tracing_subscriber::FmtSubscriber::builder() - // all spans/events with a level higher than TRACE (e.g, debug, info, warn, etc.) - // will be written to stdout. - .with_max_level(Level::DEBUG) - // builds the subscriber. - .try_init(); - #[cfg(not(debug_assertions))] - let _ = tracing_subscriber::FmtSubscriber::builder() - // all spans/events with a level higher than TRACE (e.g, debug, info, warn, etc.) - // will be written to stdout. - .with_max_level(Level::ERROR) - // builds the subscriber. - .try_init(); - }); - } -} -#[derive(Default, Clone, Debug)] -pub struct EngineTaskRegistry { - pub tasks: DashMap>, -} -impl Registry for EngineTaskRegistry { - #[instrument] - fn register(&mut self, task: Arc, identifier: Identifier) { - // Insert the task into the hashmap with (mod_id, identifier) as the key - debug!( - "TaskRegistry: Registering task {}.{}", - identifier.0, identifier.1 - ); - self.tasks.insert(identifier, task); - } - - fn get(&self, identifier: &Identifier) -> Option> { - self.tasks.get(identifier).map(|obj| obj.clone_box()) - } -} - -pub async fn clear_sled_periodically(api: Arc>, n_minutes: u64) { - info!("Lease GC started ({}m interval)", n_minutes); - let mut interval = interval(Duration::from_secs(n_minutes * 60)); - let ttl = chrono::Duration::seconds(3600); - loop { - interval.tick().await; - let api = api.read().await; - let now = Utc::now(); - let mut expired: u64 = 0; - for mut entry in api.leased_tasks.tasks.iter_mut() { - let before = entry.len(); - entry.retain(|l| now.signed_duration_since(l.given_at) < ttl); - expired += (before - entry.len()) as u64; - } - if expired > 0 { - info!("Lease GC: expired {} stale lease(s)", expired); - } - } + pub cfg: Config, // RW + pub event_bus: EventBus, // RW + pub lib_manager: LibraryManager, // RW } diff --git a/enginelib/src/config.rs b/enginelib/src/config.rs index 5f09076..7d94405 100644 --- a/enginelib/src/config.rs +++ b/enginelib/src/config.rs @@ -7,42 +7,15 @@ fn default_host() -> String { "[::1]:50051".into() } -fn default_clean_tasks() -> u64 { - 60 -} -fn default_task_block_size() -> u32 { - 0x8000 -} -fn default_pagination_limit() -> u32 { - u32::MAX -} -fn default_task_queue_size() -> u32 { - 2048 -} #[derive(Debug, Serialize, Deserialize, Clone)] pub struct ConfigTomlServer { - #[serde(default)] - pub cgrpc_token: Option, // Administrator Token, used to invoke cgrpc reqs. If not preset will default to no protection. #[serde(default = "default_host")] pub host: String, - #[serde(default = "default_clean_tasks")] - pub clean_tasks: u64, - #[serde(default = "default_pagination_limit")] - pub pagination_limit: u32, - #[serde(default = "default_task_block_size")] - pub task_block_size: u32, - #[serde(default = "default_task_queue_size")] - pub task_queue_size: u32, } impl Default for ConfigTomlServer { fn default() -> Self { Self { - cgrpc_token: None, host: default_host(), - clean_tasks: default_clean_tasks(), - pagination_limit: default_pagination_limit(), - task_block_size: default_task_block_size(), - task_queue_size: default_task_queue_size(), } } } diff --git a/enginelib/src/event.rs b/enginelib/src/event.rs index b8de754..ab58f3b 100644 --- a/enginelib/src/event.rs +++ b/enginelib/src/event.rs @@ -100,6 +100,14 @@ impl EngineEventHandlerRegistry { } impl EventBus { + pub fn has_handlers(&self, identifier: &Identifier) -> bool { + self.event_handler_registry + .event_handlers + .get(identifier) + .map(|handlers| !handlers.is_empty()) + .unwrap_or(false) + } + pub fn register_handler( &mut self, handler: H, diff --git a/enginelib/src/events/cgrpc_event.rs b/enginelib/src/events/cgrpc_event.rs index ef821a0..110768b 100644 --- a/enginelib/src/events/cgrpc_event.rs +++ b/enginelib/src/events/cgrpc_event.rs @@ -16,7 +16,7 @@ pub struct CgrpcEvent { impl CgrpcEvent { pub fn fire( - api: &mut ServerAPI, + api: &ServerAPI, handler_id: Identifier, payload: Vec, output: Arc>>, diff --git a/enginelib/src/events/mod.rs b/enginelib/src/events/mod.rs index 6b39213..de30e31 100644 --- a/enginelib/src/events/mod.rs +++ b/enginelib/src/events/mod.rs @@ -41,7 +41,7 @@ impl Events { } pub fn CgrpcEvent( - api: &mut ServerAPI, + api: &ServerAPI, handler_id: Identifier, payload: Vec, output: Arc>>, diff --git a/enginelib/src/task.rs b/enginelib/src/task.rs index 09347f1..6978d8e 100644 --- a/enginelib/src/task.rs +++ b/enginelib/src/task.rs @@ -1,45 +1,12 @@ use std::fmt::Debug; -use std::{collections::HashMap, sync::Arc}; +use std::sync::Arc; use crate::Identifier; use chrono::{DateTime, Utc}; -use crossbeam::queue::ArrayQueue; -use dashmap::{DashMap, DashSet}; +use dashmap::DashMap; use serde::{Deserialize, Serialize}; use tracing::{error, instrument, warn}; -#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] -pub struct StoredTask { - pub bytes: Vec, - pub id: String, -} -#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] -pub struct LeasedTask { - pub stored_task: Arc, - pub user_id: String, - pub given_at: DateTime, -} - -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct StoredTaskBlock { - pub tasks: Vec, -} -#[derive(Debug, Default)] -pub struct TaskQueue { - pub tasks: DashMap< - Identifier, - ( - async_channel::Receiver, - async_channel::Sender, - ), - >, -} - -#[derive(Debug, Default, Clone)] -pub struct LeasedTaskQueue { - pub tasks: DashMap>, -} - pub trait Verifiable { fn verify(&self, b: Vec) -> bool; } diff --git a/enginelib/tests/event_tests.rs b/enginelib/tests/event_tests.rs deleted file mode 100644 index 9288a3d..0000000 --- a/enginelib/tests/event_tests.rs +++ /dev/null @@ -1,221 +0,0 @@ -use enginelib::{Registry, api::ServerAPI, events::ID, task::Verifiable}; -use macros::Verifiable; -use std::sync::Arc; -use tracing_test::traced_test; - -#[derive(Clone, Debug, macros::Event)] -#[event(namespace = "test", name = "test_event", cancellable)] -struct TestEvent { - pub value: i32, - pub cancelled: bool, -} - -#[macros::event_handler(namespace = "test", name = "test_event")] -fn increment_test_event(event: &mut TestEvent) { - event.value += 1; -} - -#[derive(Clone, Debug, macros::Event)] -#[event(namespace = "test", name = "stateful_event")] -struct StatefulEvent { - pub value: u32, -} - -#[macros::event_handler(namespace = "test", name = "stateful_event", ctx = 7u32)] -fn add_ctx_to_event(event: &mut StatefulEvent, ctx: &u32) { - event.value += *ctx; -} - -#[traced_test] -#[test] -fn id() { - assert!(ID("namespace", "id") == ID("namespace", "id")) -} - -#[traced_test] -#[test] -fn test_event_registration_and_handling() { - let mut api = ServerAPI::test_default(); - - let mut test_event = TestEvent { - value: 0, - cancelled: false, - }; - - enginelib::event::register_inventory_handlers(&mut api); - api.event_bus.fire(&mut test_event); - assert_eq!(test_event.value, 1); - drop(api.db); -} - -#[traced_test] -#[test] -fn test_stateful_event_auto_registration() { - let mut api = ServerAPI::test_default(); - - enginelib::event::register_inventory_handlers(&mut api); - - let mut event = StatefulEvent { value: 0 }; - api.event_bus.fire(&mut event); - assert_eq!(event.value, 7); - drop(api.db); -} - -#[traced_test] -#[test] -fn test_task_registration() { - use enginelib::task::Task; - use serde::{Deserialize, Serialize}; - - #[derive(Debug, Clone, Serialize, Deserialize, Verifiable)] - struct TestTask { - pub value: i32, - pub id: (String, String), - } - - impl Task for TestTask { - fn to_toml(&self) -> String { - "".into() - } - fn get_id(&self) -> (String, String) { - self.id.clone() - } - fn clone_box(&self) -> Box { - Box::new(self.clone()) - } - fn run_cpu(&mut self) { - self.value += 1; - } - fn to_bytes(&self) -> Vec { - postcard::to_allocvec(self).unwrap() - } - fn from_bytes(&self, bytes: &[u8]) -> Box { - Box::new(postcard::from_bytes::(bytes).unwrap()) - } - fn from_toml(&self, d: String) -> Box { - return Box::new(self.clone()); - } - } - let mut api = ServerAPI::test_default(); - let task_id = ID("test", "test_task"); - - // Register the task type - api.task_registry.register( - Arc::new(TestTask { - value: 0, - id: task_id.clone(), - }), - task_id.clone(), - ); - - // Verify it was registered - assert!(api.task_registry.tasks.contains_key(&task_id)); - drop(api.db); -} - -#[traced_test] -#[test] -fn test_task_execution() { - use enginelib::task::{Runner, Task}; - use serde::{Deserialize, Serialize}; - - #[derive(Debug, Clone, Serialize, Deserialize, Verifiable)] - struct TestTask { - pub value: i32, - pub id: (String, String), - } - - impl Task for TestTask { - fn to_toml(&self) -> String { - "".into() - } - fn from_toml(&self, d: String) -> Box { - return Box::new(self.clone()); - } - fn get_id(&self) -> (String, String) { - self.id.clone() - } - fn clone_box(&self) -> Box { - Box::new(self.clone()) - } - fn run_cpu(&mut self) { - self.value += 1; - } - fn to_bytes(&self) -> Vec { - postcard::to_allocvec(self).unwrap() - } - fn from_bytes(&self, bytes: &[u8]) -> Box { - Box::new(postcard::from_bytes::(bytes).unwrap()) - } - } - - // Test task execution directly - let mut task = TestTask { - value: 0, - id: ID("test", "test_task"), - }; - - task.run(Some(Runner::CPU)); - assert_eq!(task.value, 1); -} - -#[traced_test] -#[test] -fn test_task_serialization() { - use enginelib::task::{StoredTask, Task}; - use serde::{Deserialize, Serialize}; - - #[derive(Debug, Clone, Serialize, Deserialize, Verifiable)] - struct TestTask { - pub value: i32, - pub id: (String, String), - } - - impl Task for TestTask { - fn to_toml(&self) -> String { - "".into() - } - fn from_toml(&self, d: String) -> Box { - return Box::new(self.clone()); - } - fn get_id(&self) -> (String, String) { - self.id.clone() - } - fn clone_box(&self) -> Box { - Box::new(self.clone()) - } - fn run_cpu(&mut self) { - self.value += 1; - } - fn to_bytes(&self) -> Vec { - postcard::to_allocvec(self).unwrap() - } - fn from_bytes(&self, bytes: &[u8]) -> Box { - Box::new(postcard::from_bytes::(bytes).unwrap()) - } - } - - let task = TestTask { - value: 42, - id: ID("test", "test_task"), - }; - - // Test serialization and deserialization - let serialized = task.to_bytes(); - let stored_task = StoredTask { - bytes: serialized, - id: "id".into(), - }; - - // Deserialize - let deserialized_task: TestTask = postcard::from_bytes(&stored_task.bytes).unwrap(); - assert_eq!(deserialized_task.value, 42); - - // Test the from_bytes function - let recreated_task = task.from_bytes(&stored_task.bytes); - // We need a way to check the value inside the recreated task - // Since we can't directly access the value, we'll serialize it again and deserialize manually - let bytes = recreated_task.to_bytes(); - let final_task: TestTask = postcard::from_bytes(&bytes).unwrap(); - assert_eq!(final_task.value, 42); -} diff --git a/rfc/rfc1006.md b/rfc/rfc1006.md new file mode 100644 index 0000000..834ed28 --- /dev/null +++ b/rfc/rfc1006.md @@ -0,0 +1,1176 @@ +# RFC 1006: Three-Engine Server Architecture + +- **Status:** Draft +- **Scope:** Near-total rewrite of the server runtime +- **Audience:** Engine maintainers, transport authors, data-backend authors, and mod authors +- **Last updated:** 2026-07-12 + +## Summary + +Replace the current server structure with three strict architectural engines: + +1. **COM Engine** — accepts requests over a communication technology such as gRPC, HTTP, a message broker, an embedded API, or a future protocol; translates between that protocol and transport-neutral server commands; applies protocol-level flow control; and maps server errors back to protocol errors. +2. **Server Engine** — implements authentication, authorization, task use cases, task validation, lifecycle policy, mod hooks, and orchestration. It contains no `tonic`, protobuf, sled, socket, or database-key knowledge. +3. **Data Engine** — owns durable task state, atomic task transitions, leasing, idempotency, pagination, and backend-specific acceleration. It contains no gRPC or server-policy knowledge. + +This is intentionally a near-rewrite. The desired result is not `EngineService` hidden behind one trait. The desired result is a server whose application semantics can run without gRPC and without sled, and whose gRPC and sled implementations can be replaced independently. + +The three engines are **logical boundaries and Rust crates**, not three mandatory network services. The first production implementation SHOULD compose them in one process. A later deployment MAY move COM or Data across a process boundary, but only after a measured need and a separately versioned internal protocol exist. + +## Decision in one diagram + +```text + external clients + | + | gRPC / HTTP / broker / embedded calls + v ++-----------------------+ +| COM Engine | protocol parsing, framing, limits, +| grpc | http | ... | context extraction, error mapping ++-----------+-----------+ + | InboundPort: transport-neutral commands/results + v ++-----------------------+ +| Server Engine | authn/authz, validation, lifecycle, +| application use cases | policies, mods, orchestration ++-----------+-----------+ + | DataPort: atomic domain operations + v ++-----------------------+ +| Data Engine | durable state machine, leases, +| sled | postgres | mem | idempotency, cursors, transactions ++-----------------------+ +``` + +Dependencies point inward: + +```text +engine-com-grpc ----> engine-server-api <---- engine-server + ^ | + | v + engine-domain <------ engine-data-api + ^ + | + engine-data-sled / engine-data-memory / future adapters +``` + +No arrow in the architecture may point from `engine-server` to a COM implementation or from a domain type to a storage implementation. + +## Terminology and normative language + +The words **MUST**, **MUST NOT**, **SHOULD**, **SHOULD NOT**, and **MAY** describe requirements for the rewrite. + +- **Task type**: the registered `(namespace, task)` pair, currently represented by `Identifier`. +- **Task instance**: one submitted unit of work with its own opaque ID. +- **Lease**: a time-bounded, exclusive claim to attempt a task. +- **Completion**: an accepted result for the current lease. +- **Engine**: a cohesive logical component with a public contract. It does not imply a process. +- **Port**: an application-owned contract. +- **Adapter**: a technology-specific implementation of a port. +- **Command**: a request to perform an application use case. + +## Motivation + +The current code is optimized around direct access to a shared `ServerAPI`. That made early development fast, but it makes communication, application policy, concurrency, and persistence change together. + +The gRPC service in `engine/src/lib.rs` currently does all of the following inside request handlers: + +- reads tonic metadata and constructs identity/authentication inputs; +- parses protobuf-shaped task identifiers; +- invokes auth and lifecycle events; +- queries the task registry and validates payloads; +- reads queue configuration; +- refills in-memory channels by scanning sled; +- creates and removes in-memory leases; +- performs sled batches and maps sled failures to tonic statuses; +- implements batching and stream framing; +- chooses which malformed or invalid batch members are silently skipped; +- constructs protobuf responses. + +`enginelib/src/api.rs` combines configuration, queue channels, lease maps, refill cursors, ID generation, task registration, event dispatch, module loading, sled access, logging setup, and background lease cleanup in one public struct. Mods receive that struct directly through the native Rust ABI. + +This has concrete consequences: + +| Current condition | Consequence | +|---|---| +| `EngineService` accepts `tonic::Request` in internal helpers | A non-gRPC adapter must fabricate tonic values or duplicate the use case. | +| Domain operations return `tonic::Status` | Application errors cannot be reused or mapped consistently by another protocol. | +| `ServerAPI` exposes `sled::Db` and queue internals | Storage cannot be replaced without changing server logic, auth events, mods, and tests. | +| Leases live in RAM while queued payloads live in sled | Restart semantics are implicit; lease ownership and persistence are not one atomic transition. | +| Queue refill, active IDs, and leases coordinate across several maps | Correctness relies on ordering spread across COM-facing code and data code. | +| Streaming RPCs buffer or batch through unary internal helpers | Transport framing has accidentally become application behavior. | +| Batch members are sometimes skipped while the overall RPC succeeds | Clients cannot reliably distinguish accepted, rejected, and retriable items. | +| Mods receive `&mut ServerAPI` via `extern "Rust"` | Every field-level redesign risks ABI/API breakage and grants mods unrestricted authority. | +| Page-number pagination scans and materializes state | Cost grows with the dataset and exposes backend behavior through the API. | + +The rewrite MUST make task lifecycle rules explicit before pursuing additional throughput optimizations. A faster implementation of ambiguous state transitions is still ambiguous. + +## Goals + +1. Replace gRPC without changing Server Engine or Data Engine code. +2. Replace sled without changing Server Engine or any COM adapter. +3. Run the complete application use cases in tests without sockets or a production database. +4. Define one authoritative task state machine with atomic transitions. +5. Make retry, idempotency, deadlines, cancellation, partial batch success, and delivery guarantees explicit. +6. Preserve high-throughput block operations without making block size part of correctness. +7. Narrow mod capabilities so mods do not depend on a god object or database handle. +8. Support gradual migration and rollback while the current gRPC API remains available. +9. Make process boundaries optional deployment choices rather than accidental code dependencies. +10. Give humans enough contracts, invariants, and acceptance criteria to implement the rewrite without rediscovering the architecture. + +## Non-goals + +- This RFC does not require three deployable services in the first release. +- This RFC does not select the final production database. It defines the contract a backend must satisfy. +- This RFC does not promise exactly-once task execution. It defines at-least-once leasing with idempotent completion. +- This RFC does not preserve source compatibility for mods that directly access `ServerAPI`. +- This RFC does not require keeping the current proto naming mistakes in a new API. The legacy adapter may preserve them. +- This RFC does not define worker-side execution architecture beyond the server-facing protocol. +- This RFC does not introduce arbitrary generic repositories for every struct. Ports are use-case and state-transition oriented. + +## Architectural principles + +### 1. Ports describe purposeful conversations + +The architecture follows the ports-and-adapters idea: technology-specific adapters translate an external interaction into an application port. A port is not “all database methods” or “all gRPC methods”; it is a coherent conversation such as submit, lease, complete, inspect, or administer tasks. + +### 2. Logical separation comes before process separation + +COM, Server, and Data MUST compile as independent boundaries. They SHOULD initially run in one process: + +- calls stay cheap and typed; +- cancellation and backpressure do not require another wire protocol; +- deployment remains simple; +- benchmarks reveal real boundary cost; +- a later split is possible because dependencies are already controlled. + +Turning each engine into a service immediately is rejected by this RFC. It would create an internal COM-to-Server protocol and possibly a Server-to-Data protocol, add new partial-failure modes, and make a communication-replacement project paradoxically depend on more communication. + +### 3. The Data Engine owns state transitions, not CRUD + +The Server Engine MUST NOT implement “read state, decide, then write state” for lifecycle transitions. The Data Engine MUST expose atomic operations such as `lease`, `complete`, `renew`, and `release_expired`. + +A generic `get_task` plus `put_task` repository is insufficient because it allows races to be reconstructed above the transaction boundary. + +### 4. The durable store is authoritative + +In-memory queues, indexes, caches, cursors, and channels MAY accelerate selection, but MUST NOT be the only record of: + +- task existence; +- current lifecycle state; +- lease holder/token and lease deadline; +- accepted completion; +- idempotency outcome. + +After a crash, correctness MUST be recoverable from durable data without reconstructing hidden facts from logs. + +### 5. External schemas are adapters, not domain models + +Generated protobuf types MUST remain inside `engine-com-grpc`. JSON, broker, or future protocol types belong to their adapters. The domain owns explicit Rust value types. Mapping code is intentional and tested. + +### 6. Errors are domain facts before protocol statuses + +The Server Engine returns a closed, transport-neutral error taxonomy. Each COM adapter maps it to its own error mechanism. A storage error is never formatted directly into a public response. + +### 7. Correctness semantics precede optimization + +Batching, caching, lock sharding, and zero-copy work may optimize a contract but MUST NOT redefine it. Every optimized Data Engine implementation must pass the same contract suite as the reference in-memory implementation. + +## Proposed workspace + +Names can change during implementation, but the dependency boundaries may not. + +| Crate | Responsibility | Allowed major dependencies | Forbidden knowledge | +|---|---|---|---| +| `engine-domain` | IDs, task values, state, commands/results, errors, invariants | `serde`, time/ID libraries if needed | tonic, prost, sled, sockets, config files | +| `engine-server-api` | inbound application port and request context | `engine-domain` | protobuf and database types | +| `engine-server` | use cases, authz, validation, policy, event orchestration | domain, server API, data API, mod host API | tonic, prost, sled keys | +| `engine-data-api` | atomic outbound data port and backend contract tests | domain | tonic, external protocol DTOs | +| `engine-data-memory` | deterministic reference/test adapter | data API | transport code | +| `engine-data-sled` | migration/compatibility backend | sled, data API | tonic, auth policy, mods | +| `engine-com-grpc` | legacy and/or v2 gRPC adapters | tonic, prost, server API | sled, queue maps, task validators | +| `engine-runtime` | configuration, dependency injection, startup, shutdown, health | all selected adapters | business rules | +| `engine-mod-api` | stable, capability-limited mod contracts | domain-safe DTOs | `ServerRuntime`, raw DB handles | +| `engine-client` | worker/client behavior | generated client adapter or protocol SDK | server persistence internals | + +The existing `engine` package MAY temporarily contain compatibility binaries, but the final `server` binary should be a thin composition root in `engine-runtime`. + +### Compile-time enforcement + +The workspace SHOULD add automated dependency checks. At minimum: + +- `engine-domain` has no dependency on any engine implementation crate; +- `engine-server` has no dependency on `tonic`, `prost`, `sled`, or a COM crate; +- COM crates have no dependency on Data implementations; +- Data crates have no dependency on Server or COM implementations; +- generated protobuf modules are not re-exported by domain/server crates. + +These constraints SHOULD be checked in CI, not left to review convention. + +## Domain model + +Stringly typed identifiers and overloaded payload fields should be replaced with explicit types. Example shapes are illustrative, not final syntax: + +```rust +pub struct TaskType { + pub namespace: Namespace, + pub name: TaskName, +} + +pub struct TaskId(pub Uuid); +pub struct LeaseId(pub Uuid); +pub struct IdempotencyKey(pub String); +pub struct PrincipalId(pub String); + +pub struct TaskInput(pub bytes::Bytes); +pub struct TaskOutput(pub bytes::Bytes); + +pub enum TaskState { + Queued, + Leased, + Completed, + Cancelled, + DeadLettered, +} +``` + +### ID rules + +- IDs MUST be generated by an injected `IdGenerator`, not by wall-clock formatting inside storage code. +- IDs MUST be globally unique across server processes if horizontal Server Engine instances are allowed. +- IDs MUST be opaque in public contracts; clients may compare or store them but must not parse routing data from them. +- UUIDv7 is RECOMMENDED because it is standardized, globally usable, and time-orderable. A different 128-bit opaque scheme is acceptable if collision and ordering properties are documented. +- Lease IDs MUST be unguessable and distinct for each acquisition attempt. + +### Task record + +The conceptual durable record contains at least: + +```text +task_id +task_type(namespace, name) +input_bytes +output_bytes? // or an output reference for large results +state +created_at +updated_at +available_at +priority +attempt +max_attempts? +lease_id? +lease_owner? +lease_deadline? +completed_at? +input_digest +output_digest? +revision // monotonically changing fencing/version value +``` + +Backend schemas may normalize or partition this representation. Observable semantics must remain the same. + +### State machine + +```text + lease expires / release + +-----------------------------+ + | | + v | + submit ------> QUEUED -------- lease --------> LEASED + | | + | cancel | complete(current lease) + v v + CANCELLED COMPLETED + + LEASED -- max attempts / policy --> DEAD_LETTERED + QUEUED -- max age / admin policy -> DEAD_LETTERED +``` + +Required invariants: + +1. A task has exactly one authoritative state. +2. A successful lease transition returns a unique `LeaseId`, increments `attempt` and `revision`, and durably records its deadline in the same atomic operation. +3. At most one unexpired lease is current for a task. +4. Only the current `LeaseId` may renew, release, or complete the task. +5. Completion atomically checks the lease token and changes `LEASED -> COMPLETED`. +6. A stale worker cannot complete a task after its lease expired and another worker acquired it. The lease ID/revision is a fencing token. +7. Repeating a successful completion with the same task ID, lease ID, and output digest returns the original success. +8. Repeating it with different output returns `Conflict`. +9. An expired lease is available for another attempt even if cleanup has not physically rewritten the row yet, provided the backend can atomically claim it. +10. Deletion MUST NOT make a concurrently leased task appear successfully completed. Administrative deletion semantics must be explicit: reject, cancel/tombstone, or force-delete with audit. + +### Delivery guarantee + +The engine promises **at-least-once delivery**: + +- a worker may receive a task and fail before completion; +- a response may be lost after a commit; +- an expired task may be leased again; +- therefore task execution may occur more than once. + +The engine does not claim exactly-once execution. It provides fencing and idempotent completion so only a valid completion is accepted. Task implementations that cause external side effects remain responsible for their own idempotency or transactional integration. + +## Server Engine + +The Server Engine owns application behavior. It is invoked through an inbound port and depends only on outbound ports. + +### Inbound request context + +Every COM adapter constructs a bounded context: + +```rust +pub struct RequestContext { + pub request_id: RequestId, + pub credential: Credential, + pub deadline: Option, + pub cancellation: Cancellation, + pub trace: TraceContext, + pub peer: Option, +} +``` + +Rules: + +- `credential` is opaque to COM except for syntax/size checks. +- Authentication and authorization are Server Engine policies through an `AuthPort`. +- The context MUST NOT expose tonic metadata maps to the Server Engine. +- Deadlines and cancellation MUST be checked before expensive work and passed to outbound operations. +- Credentials MUST NOT be copied into tracing baggage, logs, errors, or mod events. +- `request_id` is for correlation, not idempotency. Mutating commands use an explicit idempotency key. + +### Inbound port + +The public shape should be use-case oriented: + +```rust +pub trait ServerPort: Send + Sync { + async fn metadata(&self, ctx: RequestContext) -> Result; + async fn list_task_types(&self, ctx: RequestContext) -> Result, EngineError>; + async fn submit(&self, ctx: RequestContext, cmd: SubmitTasks) + -> Result; + async fn lease(&self, ctx: RequestContext, cmd: LeaseTasks) + -> Result; + async fn complete(&self, ctx: RequestContext, cmd: CompleteTasks) + -> Result; + async fn renew(&self, ctx: RequestContext, cmd: RenewLeases) + -> Result; + async fn query(&self, ctx: RequestContext, query: TaskQuery) + -> Result; + async fn administer(&self, ctx: RequestContext, cmd: AdminCommand) + -> Result; + async fn invoke_extension(&self, ctx: RequestContext, cmd: ExtensionCommand) + -> Result; +} +``` + +Implementation note: the exact Rust async-trait representation must be chosen deliberately. Native `async fn` in traits and `dyn` dispatch constraints evolve independently. The project may use generic composition, boxed futures, or a small type-erasure crate. This is an implementation choice, not a reason to leak adapters inward. + +### Use-case pipeline + +Every mutating use case follows the same conceptual sequence: + +```text +validate structural limits + -> authenticate + -> authorize action/resource + -> run cancellable before-policy/mod hooks + -> validate task-type-specific content + -> invoke one atomic Data Engine operation + -> emit post-commit domain events + -> return per-item result +``` + +Ordering deviations MUST be documented. In particular, payload validation must occur before durable submission, while lease-token validation belongs inside the atomic completion transition. + +### Batch semantics + +Blocks remain important for throughput, but vague partial success is prohibited. + +Every batch command MUST declare one of two modes: + +- `Atomic`: all valid items commit together or none commit. A structurally invalid member rejects the whole batch. +- `BestEffort`: each member returns an explicit result, and successful items may commit independently or in backend sub-batches. + +Example result: + +```rust +pub struct BatchResult { + pub items: Vec>, +} + +pub enum ItemResult { + Applied(T), + AlreadyApplied(T), + Rejected(ItemError), + Retryable(ItemError), +} +``` + +The COM adapter may choose a protocol representation, but it MUST NOT turn rejected members into silent overall success. + +### Task validation registry + +The Server Engine owns a `TaskCatalog` port: + +```rust +pub trait TaskCatalog: Send + Sync { + fn descriptor(&self, task_type: &TaskType) -> Option; + fn validate_input(&self, task_type: &TaskType, input: &[u8]) -> ValidationResult; + fn validate_output(&self, task_type: &TaskType, output: &[u8]) -> ValidationResult; +} +``` + +Data does not call mod code or understand task payloads. COM does not validate domain payloads. The catalog is built during startup and becomes immutable or versioned for the life of requests. + +### Error model + +The Server Engine MUST return structured errors with stable codes: + +```rust +pub enum EngineErrorCode { + InvalidArgument, + Unauthenticated, + PermissionDenied, + NotFound, + AlreadyExists, + Conflict, + FailedPrecondition, + ResourceExhausted, + Cancelled, + DeadlineExceeded, + Unavailable, + Internal, +} + +pub struct EngineError { + pub code: EngineErrorCode, + pub safe_message: String, + pub retry: RetryAdvice, + pub details: Vec, + pub error_id: ErrorId, +} +``` + +- `safe_message` is client-visible and MUST contain no database keys, credentials, filesystem paths, panic text, or internal module details. +- Internal sources are retained in logs/spans keyed by `error_id`. +- Retryability is explicit and may include a minimum delay. +- Adapters own protocol mapping; the domain never imports protocol status types. + +## Data Engine + +The Data Engine contract is the most important rewrite boundary because it centralizes lifecycle correctness. + +### Port shape + +The port MUST expose transitions rather than storage primitives: + +```rust +pub trait DataPort: Send + Sync { + async fn submit(&self, op: SubmitOp) -> Result; + async fn lease(&self, op: LeaseOp) -> Result, DataError>; + async fn complete(&self, op: CompleteOp) -> Result; + async fn renew(&self, op: RenewOp) -> Result; + async fn release(&self, op: ReleaseOp) -> Result; + async fn query(&self, op: QueryOp) -> Result, DataError>; + async fn administer(&self, op: DataAdminOp) -> Result; + async fn health(&self) -> DataHealth; +} +``` + +The port MUST NOT expose: + +- raw sled/database handles; +- backend key construction; +- iterators whose lifetime is tied to backend guards; +- channels or `DashMap` references; +- a transaction object that lets the Server Engine perform arbitrary reads/writes; +- protocol response types. + +### Atomic operations + +#### Submit + +`submit` atomically: + +1. checks the idempotency key in the command scope; +2. allocates or accepts Server-generated task IDs according to one documented rule; +3. inserts queued tasks; +4. records the idempotency result; +5. makes tasks visible to lease selection only after commit. + +If the client retries after a lost response, the same idempotency key and same command digest return the same task IDs. The same key with a different digest returns `Conflict`. + +#### Lease + +`lease` atomically selects up to `limit` eligible tasks and changes their state to leased. It records the owner, unique lease ID, attempt number, deadline, and revision before returning payloads. + +Selection MAY consider task type, priority, `available_at`, and fairness. Ordering is a documented policy; it is not inferred from backend key order. + +The lease duration has server-configured minimum and maximum bounds. A client-provided value outside those bounds is rejected or clamped according to one documented API rule. + +#### Complete + +`complete` atomically: + +1. finds the task; +2. verifies that it is leased; +3. verifies the current lease ID and revision; +4. verifies that completion is allowed at the transaction's notion of time; +5. applies idempotency rules; +6. stores output or its reference; +7. transitions to completed; +8. records a post-commit event/outbox item if configured. + +The Server Engine validates output content before this call, but the Data Engine enforces ownership and lifecycle invariants inside the transaction. + +#### Renew and expiration + +Lease expiration MUST use a backend-consistent time source. If multiple server instances share a backend, correctness MUST NOT depend on their wall clocks being perfectly synchronized. + +Expiration may be lazy during `lease` selection, eager via a sweeper, or both. A sweeper is an optimization and observability mechanism, not the sole correctness path. + +### Queue acceleration + +The current in-memory block channel can be retained only as a Data Engine implementation detail. A correct adapter follows these rules: + +1. The cache contains references/IDs or immutable snapshots, never exclusive ownership facts. +2. An item removed from the cache is not leased until the durable atomic lease succeeds. +3. Duplicate cache entries are tolerated and deduplicated by the durable transition. +4. Cache loss only affects performance. +5. Cache refill policy is invisible to the Server and COM engines. +6. Requested COM block size does not reconfigure global Data Engine queue structure. + +This permits a simple database-first reference adapter and a later high-throughput indexed adapter behind the same tests. + +### Pagination + +Page-number pagination SHOULD be replaced with opaque cursor pagination: + +```text +query(filter, sort, limit, after_cursor) -> items + next_cursor +``` + +- cursors are adapter-issued and opaque to clients; +- a cursor includes or authenticates the filter/sort context; +- maximum page size is enforced by Server policy and COM byte limits; +- consistency semantics are documented as snapshot or forward scan; +- cursor decoding errors map to `InvalidArgument`, not internal storage errors. + +The legacy gRPC adapter may emulate page numbers temporarily, with known performance limitations and a configured maximum reachable page. + +### Backend contract suite + +`engine-data-api` MUST ship a reusable conformance suite that every backend runs. It includes: + +- concurrent lease attempts never both receive a current lease for the same task; +- stale lease completion fails after re-lease; +- duplicate submit returns identical IDs; +- conflicting idempotency-key reuse fails; +- duplicate completion is idempotent; +- completion with changed output conflicts; +- expiry makes work available without depending on a sweeper tick; +- cancellation/deletion races obey the state machine; +- process-reopen tests preserve state and idempotency for durable adapters; +- cursor traversal has no duplicates or omissions under its documented consistency model; +- batch atomic/best-effort modes match their declaration; +- injected failures at commit boundaries do not expose half transitions. + +The in-memory adapter is the readable semantic reference. It is not automatically the production performance reference. + +## COM Engine + +COM is an adapter layer, not a second application server. + +### Responsibilities + +Every COM adapter MUST: + +- listen/connect using its technology; +- decode and structurally validate frames/messages; +- enforce message, stream, concurrency, and connection limits before allocating unbounded memory; +- extract credentials, peer information, deadlines, cancellation, request IDs, and trace context; +- map protocol DTOs to domain commands; +- call `ServerPort` only; +- map domain results and errors to protocol responses; +- enforce protocol-native backpressure; +- stop accepting new work during drain and allow bounded in-flight completion; +- emit adapter-level metrics without duplicating business metrics. + +It MUST NOT: + +- access a Data Engine or database; +- call task validators directly; +- decide authorization; +- create or remove leases; +- fire business lifecycle events; +- silently reinterpret partial failures; +- put tonic/protobuf/HTTP/broker types into Server commands. + +### gRPC adapter + +`engine-com-grpc` replaces the application logic currently in `engine/src/lib.rs` with mapping code. + +Legacy method mapping may be: + +| Existing RPC | Server Engine operation | Compatibility note | +|---|---|---| +| `CreateTaskBlock` | `submit(return_items=true)` | Preserve v1 response shape. | +| `EnqueueTaskBlock` | `submit(return_items=false)` | Return only after durable acceptance. | +| `AquireTaskBlock` | `lease` | Preserve misspelling in v1 only. | +| `AquireTaskStream` | `lease`, then frame items | Streaming is representation, not a separate lifecycle operation. | +| `PublishTaskBlock` | `complete` | v1 lacks lease tokens and needs a constrained compatibility policy. | +| `PublishTaskStream` | bounded stream-to-batch `complete` calls | Must report terminal failure consistently. | +| `AquireTaskReg` | `list_task_types` | Preserve v1 shape. | +| `GetTasks` | `query` | Page-to-cursor emulation is temporary. | +| `DeleteTaskBlock` | `administer` | Explicit state/precondition mapping. | +| `CheckAuth` | an auth/admin probe | Must not receive a DB handle. | +| `GetMetadata` | `metadata` | Adapter adds protocol version only if appropriate. | +| `cgrpc` | `invoke_extension` | Rename and redesign in v2. | + +### Error mapping for gRPC + +The adapter owns a documented mapping, for example: + +| Engine error | gRPC status | +|---|---| +| `InvalidArgument` | `INVALID_ARGUMENT` | +| `Unauthenticated` | `UNAUTHENTICATED` | +| `PermissionDenied` | `PERMISSION_DENIED` | +| `NotFound` | `NOT_FOUND` | +| `AlreadyExists` | `ALREADY_EXISTS` | +| `Conflict` / stale lease | `ABORTED` or `FAILED_PRECONDITION`, selected consistently by detail code | +| `ResourceExhausted` | `RESOURCE_EXHAUSTED` | +| `Cancelled` | `CANCELLED` | +| `DeadlineExceeded` | `DEADLINE_EXCEEDED` | +| transient Data failure | `UNAVAILABLE` | +| unexpected fault | `INTERNAL` with safe error ID | + +Mappings are tested as adapter conformance tests. Server code never constructs `tonic::Status`. + +### Deadlines and cancellation + +The gRPC adapter MUST extract client deadlines and cancellation. The Server Engine and Data Port receive the remaining budget/cancellation signal. Work spawned for a request must not become detached merely because the client disconnected. + +For state-changing operations, a deadline response is inherently ambiguous if commit raced with timeout. Idempotency keys are therefore required for safe retry. The adapter documentation MUST say that `DEADLINE_EXCEEDED` does not prove a mutation did not commit. + +### Streaming and backpressure + +- Incoming streams are read into bounded chunks; never `collect()` an unbounded stream. +- Chunk size is a COM configuration constrained by Server maximum batch size and Data backend limits. +- Outgoing streams use bounded channels and stop producing when cancellation fires. +- If a protocol supports acknowledgement per item, the adapter maps `BatchResult` without losing detail. +- If a legacy protocol cannot represent partial results, its compatibility mode MUST choose atomic behavior or fail the whole operation; it must not lie with a blanket success. +- Compression ratios and decompressed byte limits are enforced to prevent small frames expanding without bound. + +### Adding another communication mechanism + +A new COM adapter is complete when it: + +1. depends on `engine-server-api` and `engine-domain`, not Server implementation details; +2. maps every supported operation and error explicitly; +3. passes shared COM semantic tests; +4. has bounded load/backpressure behavior; +5. propagates request context; +6. has protocol-specific integration tests; +7. can be selected in `engine-runtime` configuration without recompiling Server or Data code, if dynamic selection is a project requirement. + +An HTTP adapter, for example, should contain routes and JSON DTOs only; it should invoke the same `ServerPort` instance as gRPC in a dual-listener process. + +## Authentication and authorization + +Auth is event-driven today and can receive a cloned sled handle. That contract must end. + +### Proposed ports + +```rust +pub trait Authenticator: Send + Sync { + async fn authenticate(&self, credential: &Credential, peer: Option<&PeerIdentity>) + -> Result; +} + +pub trait Authorizer: Send + Sync { + async fn authorize(&self, principal: &Principal, action: Action, resource: Resource) + -> Result; +} +``` + +- COM extracts but does not interpret protocol-carried credentials beyond safe syntax limits. +- Server authenticates and authorizes each operation. +- Data trusts the Server Engine call boundary in-process. If Data later becomes remote, service identity and authorization need a separate threat model. +- A missing credential is `Unauthenticated`; an identified principal lacking access is `PermissionDenied`. +- Default-allow authentication is forbidden in production profiles. Development bypasses must be explicit, loud in logs/health, and impossible to enable accidentally through a missing token. +- Admin and user actions use the same policy vocabulary, with distinct permissions. + +## Mods and events + +The current mod ABI conflicts with the new boundaries because `run(api: &mut ServerAPI)` exposes every subsystem and ties modules to Rust layout and compiler compatibility. + +### Required architectural change + +Mods MUST receive capability-limited interfaces, not `ServerAPI`, `sled::Db`, queue maps, or a COM service. + +Suggested capabilities: + +- register a task validator/descriptor; +- subscribe to specified lifecycle events; +- make bounded namespaced key/value reads and writes through a mod storage capability; +- register an extension command handler; +- emit structured telemetry; +- read a documented subset of immutable configuration. + +Capabilities are granted from metadata/config and default to least privilege. + +### Event phases + +Events are classified: + +- **Before-policy events**: synchronous, cancellable, run before the Data operation, may validate or transform explicitly mutable fields. +- **Committed domain events**: immutable facts returned/emitted only after the Data transition commits. +- **Delivery events**: asynchronous observations delivered through an outbox if loss across crashes is unacceptable. + +Post-success events MUST NOT be fired before durable commit. A failed observer must not roll back an already committed task unless the use case explicitly implements a compensating action. + +### ABI strategy + +The rewrite SHOULD replace `extern "Rust"` mods with a versioned interface definition. WebAssembly components with WIT are a strong candidate because WIT describes typed component contracts independently of Rust struct layout. This is a major compatibility decision and deserves its own RFC before implementation. + +During migration, a `LegacyModHost` MAY adapt a restricted subset of old events, but it MUST NOT recreate broad mutable access to the new runtime. Mods that depend on raw `ServerAPI` fields will require a major-version port. + +## Runtime and deployment + +### Composition root + +`engine-runtime` performs only construction and lifecycle management: + +```text +load and validate config + -> construct telemetry + -> open Data adapter and run schema checks/migrations + -> load task catalog and mod capabilities + -> construct Authenticator/Authorizer + -> construct ServerEngine + -> construct selected COM adapters with Arc + -> start readiness + -> serve until shutdown + -> stop readiness / stop accepting + -> drain with deadline + -> flush Data/telemetry and exit +``` + +No binary should mutate a partially initialized public API object. + +### Recommended initial topology + +```text +one OS process + engine-runtime + grpc adapter --------+ + optional http -------+--> one ServerEngine --> one DataPort implementation +``` + +Multiple COM adapters MAY listen simultaneously and share one Server Engine. This directly proves transport independence. + +### Optional future split + +COM may move to an edge process when independent scaling, network isolation, language diversity, or public-edge security justifies it. Doing so requires a new, private, versioned adapter between the remote COM process and `ServerPort`. The public client gRPC schema SHOULD NOT automatically become this internal protocol. + +Data may become remote only when a real Data service—not merely a database connection pool—is warranted. Most database-backed implementations should initially remain libraries that connect directly to the database. Avoid a pass-through “Data microservice” that adds latency but owns no meaningful semantics. + +### Horizontal scale + +The architecture permits multiple stateless Server/COM instances only if the selected Data adapter provides shared atomic lease semantics and the task catalog/policies are compatible across instances. + +Rolling deployments require: + +- version-compatible domain/data schemas; +- stable task validator versions or an explicit task schema version stored with each task; +- no process-local lease truth; +- backward-compatible COM API changes; +- a readiness gate until mods, schemas, and task catalog are loaded. + +## Configuration + +Configuration should be typed by owner: + +```text +[runtime] shutdown_grace, environment +[com.grpc] listen, max_frame_bytes, stream_chunk, concurrency, tls +[server] max_batch, lease_min, lease_default, lease_max, auth_policy +[data] adapter, durability, pool/cache/backend options +[mods] directories, capabilities, compatibility_mode +[telemetry] exporter, sampling, service_name +``` + +- Each engine receives only its validated config type. +- Environment overrides are parsed once in the runtime. +- Invalid production configuration fails startup with actionable errors; no `unwrap()`. +- Secrets use a secret provider/type and redact on `Debug`. +- Backend tuning such as sled cache bytes does not appear in Server or COM configuration objects. + +## Observability + +The boundary is only operable if one request can be followed across adapters and data operations. + +### Tracing + +- COM extracts or creates trace/request context. +- Server creates spans named by use case, not RPC method. +- Data creates backend-operation spans without recording payloads or credentials. +- Context is propagated across async tasks and future process boundaries. +- Untrusted incoming trace/baggage values are size-limited and sanitized; baggage must not carry secrets. + +Minimum span attributes include operation, task type where safe, batch count, result code, adapter name, and backend name. Task payload and auth material are forbidden. + +### Metrics + +At minimum: + +```text +engine_com_requests_total{adapter,operation,result} +engine_com_request_duration_seconds{adapter,operation} +engine_com_inflight{adapter,operation} +engine_server_operations_total{operation,result} +engine_server_operation_duration_seconds{operation} +engine_data_operations_total{backend,operation,result} +engine_data_operation_duration_seconds{backend,operation} +engine_tasks_submitted_total{task_type} +engine_tasks_leased_total{task_type} +engine_tasks_completed_total{task_type} +engine_lease_expired_total{task_type} +engine_task_queue_age_seconds{task_type} +engine_task_attempts{task_type} +engine_mod_hook_duration_seconds{mod,hook} +``` + +IDs such as task, user, lease, and request MUST NOT be metric labels because of cardinality. + +### Health + +- **Liveness**: process/event loop is functioning. It must not fail merely because a dependency is transiently down. +- **Readiness**: selected COM listener can serve and required Server/Data/catalog dependencies are ready. +- **Degraded status**: optional detail for mod failures, sweeper lag, or telemetry export without making every condition an outage. + +## API versioning + +External API versioning is independent from Rust crate versions, mod ABI versions, and Data schema versions. + +### Legacy gRPC v1 + +The current proto remains supported by a compatibility adapter during migration. It preserves wire field numbers and existing RPC names, including `Aquire`, while fixing only behavior that is explicitly declared a bug/security issue. + +### New API v2 + +A v2 API SHOULD: + +- use `Acquire`, `Submit`, and `Complete` terminology; +- include lease ID/revision in acquired and completed messages; +- include explicit idempotency keys for mutations; +- represent per-item batch results; +- distinguish task input and output; +- use cursor pagination; +- expose lease renewal/release if workers need them; +- define maximums and delivery semantics; +- reserve removed protobuf field numbers and names; +- expose capability/API versions in metadata. + +Do not make the domain structs serializable “as the protocol” and assume that is versioning. Each adapter owns stable schemas and conversion tests. + +## Migration plan + +This is a strangler migration around the current service, not a single unreviewable replacement commit. + +### Phase 0 — Ratify semantics and characterize behavior + +Before moving code: + +1. Approve the task state machine, at-least-once guarantee, stale-lease behavior, batch modes, auth defaults, and idempotency scope. +2. Add black-box characterization tests for every existing RPC, including malformed IDs, invalid payloads, mixed task types, partial batches, auth failure, duplicate publish, restart, stream cancellation, and backend failure. +3. Record benchmark baselines for submit, lease, complete, and end-to-end gRPC at representative payload/block/concurrency sizes. +4. Separate benchmark data directories and ensure benchmarks actually exercise concurrency and durable behavior. Simulated counter-only concurrency is not an architectural baseline. +5. Inventory which mods access which `ServerAPI` fields. + +Exit criterion: maintainers can state which legacy behaviors are compatibility requirements and which are bugs to remove. + +### Phase 1 — Introduce domain types and errors + +1. Add `engine-domain` with IDs, inputs/outputs, state, batch results, and error codes. +2. Add lossless conversion between current protobuf/task types and domain types in the legacy adapter area. +3. Keep existing handlers operational. +4. Add mapping and property tests; no persistence change yet. + +Exit criterion: no new application code accepts protobuf types outside COM conversion code. + +### Phase 2 — Extract the Data Engine behind current gRPC + +1. Add `engine-data-api` and the backend contract suite. +2. Implement `engine-data-memory` first to make semantics readable. +3. Implement `engine-data-sled` over current data, initially with compatibility behavior. +4. Move key construction, scans, batches, lease maps, refill cursors, active IDs, and cleanup out of `ServerAPI`/gRPC handlers. +5. Persist leases and idempotency, introducing a versioned schema. +6. Route current handlers through `DataPort` while responses remain unchanged. + +Exit criterion: gRPC code cannot access sled, queues, lease maps, or backend errors. + +### Phase 3 — Build the Server Engine + +1. Add `engine-server-api` and `engine-server`. +2. Move authn/authz, validation, task catalog, policy hooks, and lifecycle orchestration into use cases. +3. Return structured domain results/errors. +4. Make clocks, ID generation, auth, Data, catalog, and event publication injected dependencies. +5. Run all Server tests against `engine-data-memory` without tonic or sockets. + +Exit criterion: submit/lease/complete/query/admin behavior is fully testable by calling `ServerPort`. + +### Phase 4 — Reduce gRPC to a COM adapter + +1. Create `engine-com-grpc`. +2. Move generated proto code and tonic status mapping there. +3. Implement bounded streaming, deadline/cancellation propagation, limits, and graceful shutdown. +4. Run legacy black-box tests unchanged where behavior is intentionally compatible. +5. Add an in-process COM adapter or CLI test harness to prove transport neutrality. + +Exit criterion: changing Server/Data implementation does not rebuild protocol-neutral consumers, and a non-gRPC adapter completes the full task lifecycle. + +### Phase 5 — Data schema migration + +Choose one of these explicitly: + +- **Offline migration:** stop writes, back up, transform old keys to the new task/lease/idempotency schema, verify counts/digests, then start the new runtime. Recommended if downtime is acceptable. +- **Versioned online migration:** deploy code that can read old and new formats, write the new format, backfill with checkpoints, verify, then remove old reads in a later release. + +Avoid uncontrolled dual writes. A crash between independent old/new writes creates divergence unless both are covered by one backend transaction. Shadow reads and result comparison are safe; shadow mutation requires a designed replication/transaction mechanism. + +Required migration artifacts: + +- backup and restore procedure; +- version marker and resumable checkpoints; +- pre/post record counts by task state/type; +- payload digest verification; +- explicit mapping for current RAM leases (normally drain or expire before cutover); +- rollback rules that state when rollback becomes impossible after new writes. + +### Phase 6 — Mod API migration + +1. Freeze new uses of raw `ServerAPI` in mods. +2. Define capabilities and versioned task/event DTOs in a separate RFC. +3. Implement the new host and compatibility report tooling. +4. Port first-party mods. +5. Disable legacy native mods by default in the new major version. + +Exit criterion: no production mod can access a Data implementation or COM internals. + +### Phase 7 — API v2 and additional COM adapters + +1. Add the corrected v2 schema with lease/idempotency semantics. +2. Implement at least one genuinely different adapter or the embedded adapter as a conformance proof. +3. Document client upgrade and v1 deprecation windows. +4. Compare adapter behavior through a shared scenario suite. + +### Phase 8 — Remove legacy runtime + +Delete only after: + +- migrated data is verified; +- first-party clients/mods are ported; +- legacy adapter usage is below the agreed threshold; +- rollback window has closed; +- performance and fault tests meet gates. + +Targets for removal include public mutable `ServerAPI`, direct sled access outside Data, tonic types outside COM, and raw Rust-ABI runtime access from mods. + +## Mapping current files to the target + +| Current location | Target | +|---|---| +| `engine/src/bin/server.rs` | `engine-runtime` composition root | +| `engine/src/lib.rs` tonic service | mapping/framing in `engine-com-grpc`; use cases in `engine-server` | +| `engine/proto/engine.proto` | legacy schema owned by `engine-com-grpc`; v2 schema alongside it | +| `enginelib/src/api.rs` sled methods | `engine-data-sled` | +| `enginelib/src/api.rs` queue/refill/lease state | Data adapter internals or removed in database-first implementation | +| `enginelib/src/api.rs` registry/event/config fields | Server catalog/event host/typed configuration owners | +| `enginelib/src/task.rs` stored task/lease types | domain types; execution-specific `Task` trait moves to mod/worker SDK | +| `enginelib/src/event.rs` and `events/*` | capability-limited Server event host and immutable post-commit events | +| `enginelib/src/plugin.rs` | versioned mod host, eventually without `extern "Rust" ServerAPI` | +| `engine/tests/task_streaming_tests.rs` | COM gRPC integration tests plus protocol-neutral Server scenario tests | +| `engine/src/bin/no_grpc.rs` | real Server/Data benchmark, not direct internal-map manipulation | + +## Testing strategy + +### Test pyramid + +1. **Domain tests:** state transition tables, ID parsing boundaries, error/detail rules. +2. **Server use-case tests:** fake clock, fake auth, fake catalog, memory Data; no network. +3. **Data contract tests:** identical suite against memory, sled compatibility, and every future backend. +4. **COM conformance tests:** shared scenarios invoked through adapter-specific clients. +5. **gRPC integration tests:** metadata, statuses, deadlines, cancellation, streaming, reflection, limits. +6. **Fault tests:** crash/reopen, delayed commits, lost response simulation, cancellation at every await, expired leases, poisoned/unavailable mods. +7. **Property/model tests:** random operation sequences compared with a small reference state machine. +8. **Performance tests:** per-engine and end-to-end throughput/latency/resource measurements. + +### Required race tests + +- N workers lease one task concurrently; exactly one current lease wins. +- Old and new lease holders complete concurrently; only the new/current lease can first complete. +- Completion commits while the COM deadline expires; retry with idempotency returns the committed result. +- Delete/cancel races with lease and complete follow a declared ordering. +- Two Server instances submit the same idempotency key concurrently. +- Process restart occurs after durable lease commit but before response. +- Stream disconnect occurs mid-batch in atomic and best-effort modes. + +### Performance gates + +Numeric gates must be ratified from Phase 0 rather than invented here. The release gate MUST include: + +- p50/p95/p99 latency and throughput for submit/lease/complete; +- end-to-end gRPC and protocol-neutral baselines; +- CPU, allocations, memory, disk growth, and write amplification; +- empty queue and deep backlog behavior; +- 1, 8, and high-concurrency worker cases; +- small and large payloads; +- cache-cold restart behavior; +- correctness checks enabled during the benchmark. + +No result may be extrapolated from a small test to one billion tasks as an acceptance substitute. + +## Security considerations + +- Remove permissive auth behavior from production defaults. +- Bound decoded message count, individual payload bytes, aggregate batch bytes, stream duration, and in-flight work per principal/connection. +- Treat task namespace/name, IDs, cursor, metadata, and mod extension inputs as untrusted. +- Do not expose backend errors or paths. +- Lease IDs and idempotency keys are sensitive capability-like values and must not be metric labels. +- Authorize queries and admin operations by task namespace/type, not merely by endpoint. +- Mods execute with explicit capabilities and resource budgets; a mod must not block the async runtime indefinitely. +- If COM is remote from Server, authenticate the COM service and do not trust caller-supplied principals without cryptographic/channel identity. +- If Data is remote, encrypt/authenticate the link and re-evaluate which invariant checks remain server-side versus data-side. +- Trace/baggage propagation is sanitized at trust boundaries. + +## Failure and shutdown semantics + +| Failure | Required behavior | +|---|---| +| COM listener failure | Other listeners MAY continue; readiness reflects configured required listeners. | +| Client disconnect | Cancel request-scoped work where safe; committed mutations remain committed. | +| Data transient failure | Return structured retryable `Unavailable`; do not report success. | +| Data ambiguous commit | Retry via idempotency; do not blindly replay with a new key. | +| Mod before-hook failure | Follow configured fail-open/fail-closed policy per hook; auth/security hooks default fail-closed. | +| Post-commit observer failure | Record/retry through outbox if durable delivery is configured; never pretend commit failed. | +| Lease sweeper failure | Lazy expiry keeps correctness; readiness/alerts expose lag if it risks capacity. | +| Panic in request task | Convert at outer boundary to internal error, record error ID, keep process alive where unwind safety permits. | +| Shutdown | Stop readiness/acceptance, cancel or drain bounded in-flight calls, close adapters, flush Data/telemetry. | + +## Alternatives considered + +### A. Put a trait around the current `EngineService` + +Rejected. It would hide but preserve tonic requests, shared `ServerAPI`, mixed persistence and policy, and non-atomic lease behavior. A mock of a coupled object is still coupled architecture. + +### B. Extract only COM and leave Server/Data together + +This would meet the narrow goal of adding HTTP, but storage and lifecycle correctness would remain inseparable. Given the rewrite scope, the Data boundary is worth defining now. Implementation may still land COM and Data extraction in phases. + +### C. Make COM, Server, and Data three microservices immediately + +Rejected for the initial topology. It adds operational and consistency cost without being necessary for replaceability. The logical boundaries preserve this future option. + +### D. Use a message broker as the whole Data Engine + +Not selected as the universal model. Brokers can provide excellent delivery primitives, but the engine also needs task metadata, state queries, admin operations, results, idempotency, and mod-defined task catalogs. A broker-backed Data adapter MAY be built if it satisfies the entire contract and documents its consistency limitations. + +### E. Adopt full event sourcing + +Deferred. An append-only event log could improve audit/rebuild behavior, but it makes projections, schema evolution, storage volume, and operational recovery central to the rewrite. The proposed committed events/outbox preserve an evolution path without requiring all state to be reconstructed from events today. + +### F. Keep public `ServerAPI` as a compatibility facade forever + +Rejected. It would become a permanent escape hatch around every boundary. A temporary facade may exist only with a removal milestone and no access to new internals. + +### G. Select PostgreSQL now and design directly around SQL + +Not required by this RFC. PostgreSQL is a strong candidate for shared, transactional multi-server leasing; an embedded backend may be preferable for simple deployments. The contract suite should make the choice evidence-based. SQL types and queries must remain inside the adapter either way. + +## Risks and mitigations + +| Risk | Mitigation | +|---|---| +| Rewrite stalls while features continue landing | Phase by endpoint/use case; freeze new direct dependencies; keep legacy adapter running. | +| Abstraction harms throughput | Batch-oriented ports, benchmark each boundary, static/generic composition where useful, optimize adapters behind contracts. | +| Data abstraction becomes lowest-common-denominator CRUD | Define semantic atomic operations and explicit backend capability checks. | +| Legacy gRPC cannot express new lease/idempotency semantics | Constrained v1 compatibility policy plus v2; do not weaken internal invariants indefinitely. | +| Mod ecosystem breaks | Inventory usage, capability RFC/tooling, compatibility report, major-version communication. | +| Dual storage migration corrupts data | Prefer offline migration or transactional/versioned online plan; digest verification and checkpoints. | +| Three “engines” become three god objects | Split ports by purposeful conversation internally; keep facade small; enforce dependencies. | +| Too many tiny crates slow development | Use crates only for enforced architectural boundaries; modules may organize internals. | +| In-memory optimization reintroduces split-brain truth | Durable state remains authoritative; cache duplication/loss covered by contract tests. | +| Error compatibility changes clients | Golden status/detail tests and versioned API mapping. | + +## Acceptance criteria + +The rewrite described by this RFC is complete when all of the following are true: + +1. The complete submit → lease → complete lifecycle runs through `ServerPort` with the memory Data adapter and no tonic/sled dependency. +2. The same Server Engine instance can be driven by gRPC and at least one non-gRPC/embedded COM adapter. +3. Swapping memory/sled/another Data adapter requires no Server or COM source changes beyond runtime composition/configuration. +4. All Data adapters pass the shared atomicity, lease, idempotency, expiry, pagination, and reopen contract tests applicable to their durability class. +5. No Server Engine source imports tonic, prost, sled, generated protobuf types, or backend keys. +6. No COM source accesses Data directly. +7. Leases are durable and fenced; stale completion is rejected. +8. Mutating operations have documented idempotency and ambiguous-timeout behavior. +9. Batches return explicit item outcomes or are explicitly atomic. +10. Production auth is not default-allow, and auth code has no raw database handle. +11. Mods use capabilities rather than mutable runtime/database access, or legacy mods are clearly isolated and disabled by default for the new major version. +12. Graceful shutdown, deadline, cancellation, resource-limit, and trace-context behavior is tested. +13. Data migration and rollback procedures have been exercised on a production-sized copy. +14. Workspace tests, builds, lint policy, fault tests, and ratified performance gates pass. +15. Architecture dependency rules run in CI. + +## Decisions required before implementation + +These are blocking architecture decisions, not details to leave to the first implementer: + +1. Is the first production Data backend a compatibility sled adapter, a new embedded backend, PostgreSQL, or two tiers? +2. What exact lease expiry rule applies when completion and expiry race? +3. What are default/min/max lease durations and maximum attempts? +4. Are submit and complete batches atomic or best-effort by default in v2? +5. What is the idempotency scope and retention period: principal + operation + key, tenant + key, or global key? +6. Does task output stay inline, move to blob storage above a threshold, or remain application-defined bytes? +7. What ordering/fairness promise does lease selection make? +8. Which legacy partial-success behaviors are bugs versus compatibility requirements? +9. How long is gRPC v1 supported, and how can its lack of lease tokens be safely constrained? +10. Is WASM/WIT adopted for mods, or is another stable interface chosen? +11. Which hooks require durable post-commit delivery through an outbox? +12. Which COM listeners are required for readiness when several run together? + +## Recommended first implementation slice + +Do not start by moving files. Start with one vertical slice: + +1. Add domain `TaskType`, `TaskId`, `LeaseId`, `TaskInput`, `TaskOutput`, batch outcomes, and errors. +2. Specify `submit`, `lease`, and `complete` only. +3. Implement their memory Data state machine and conformance tests. +4. Implement Server use cases with fake auth/catalog/clock. +5. Adapt only `CreateTaskBlock`, `AquireTaskBlock`, and `PublishTaskBlock` to that Server port behind a feature/config switch. +6. Add an embedded test adapter that calls the same port. +7. Run characterization, race, restart, and performance comparisons. + +This slice proves all three boundaries before the team migrates queries, streams, admin RPCs, extensions, and mods. + +## Research basis + +This design is informed by the following primary or official sources: + +- Alistair Cockburn's original [Hexagonal Architecture / Ports and Adapters](https://alistair.cockburn.us/hexagonal-architecture) describes technology-specific adapters around purposeful application ports and explicitly includes alternate databases and test adapters. +- The official gRPC guides document [deadlines](https://grpc.io/docs/guides/deadlines/), [cancellation](https://grpc.io/docs/guides/cancellation/), [status codes](https://grpc.io/docs/guides/status-codes/), and [graceful shutdown](https://grpc.io/docs/guides/server-graceful-stop/). These behaviors belong in COM context propagation and mapping, while the Server must cooperate with cancellation. +- The official [Protocol Buffers proto3 guide](https://protobuf.dev/programming-guides/proto3/) documents wire-safe evolution and the need to reserve removed field numbers/names. +- OpenTelemetry's official [context propagation](https://opentelemetry.io/docs/concepts/context-propagation/) and [propagator specification](https://opentelemetry.io/docs/specs/otel/context/api-propagators/) motivate a protocol-neutral request context and warn about trust-boundary/baggage handling. +- [RFC 9562](https://www.rfc-editor.org/rfc/rfc9562.html) standardizes UUIDv7 and its time-ordered layout. +- The Bytecode Alliance's [WIT reference](https://component-model.bytecodealliance.org/design/wit.html) explains WIT as a language for typed component contracts; it supports, but does not by itself decide, the proposed mod ABI direction. +- The Rust Reference's [dyn compatibility rules](https://doc.rust-lang.org/reference/items/traits.html#dyn-compatibility) are why the RFC leaves generic versus erased async port dispatch as an explicit implementation decision. + +## Relationship to previous RFCs + +- RFC 1002's `.rf` packaging may remain as a container during migration, but its native `run(&mut EngineAPI)` ABI conflicts with this RFC and requires a successor mod-interface RFC. +- RFC 1003 client hooks remain conceptually client-side; protocol DTO changes in v2 require updated client mappings. +- RFC 1004 server lifecycle events are preserved as use-case concepts, but their firing location and post-commit semantics move into the Server Engine. +- RFC 1005's performance goals become Data Engine implementation work constrained by the state-machine and contract tests in this RFC. + From 617026685b34172a00d71d673c2d1f5045f5b589 Mon Sep 17 00:00:00 2001 From: Styly Date: Sat, 18 Jul 2026 13:29:34 +0100 Subject: [PATCH 19/28] rewrite: finish cleanup --- enginelib/src/api.rs | 1 - enginelib/src/events/admin_auth_event.rs | 100 +++++++++++------------ enginelib/src/events/mod.rs | 6 +- enginelib/src/prelude.rs | 2 +- 4 files changed, 54 insertions(+), 55 deletions(-) diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index 1ad6376..2e6554d 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -8,7 +8,6 @@ use crate::{ config::Config, event::{EngineEventHandlerRegistry, EventBus}, plugin::LibraryManager, - task::{LeasedTaskQueue, StoredTask, StoredTaskBlock, Task, TaskQueue}, }; pub use postcard; pub use postcard::from_bytes; diff --git a/enginelib/src/events/admin_auth_event.rs b/enginelib/src/events/admin_auth_event.rs index a1ce7d2..c4bb152 100644 --- a/enginelib/src/events/admin_auth_event.rs +++ b/enginelib/src/events/admin_auth_event.rs @@ -1,56 +1,56 @@ -use std::sync::{Arc, RwLock}; +// use std::sync::{Arc, RwLock}; -use macros::{Event, event_handler}; -use sled::Db; +// use macros::{Event, event_handler}; +// use sled::Db; -use crate::{Identifier, api::ServerAPI}; +// use crate::{Identifier, api::ServerAPI}; -#[derive(Clone, Debug, Event)] -#[event(namespace = "core", name = "admin_auth_event", cancellable)] -pub struct AdminAuthEvent { - pub cancelled: bool, - pub id: Identifier, - pub payload: String, - pub target: Identifier, - pub db: Db, - pub output: Arc>, -} -// Event trait auto-implemented by derive macro +// #[derive(Clone, Debug, Event)] +// #[event(namespace = "core", name = "admin_auth_event", cancellable)] +// pub struct AdminAuthEvent { +// pub cancelled: bool, +// pub id: Identifier, +// pub payload: String, +// pub target: Identifier, +// pub db: Db, +// pub output: Arc>, +// } +// // Event trait auto-implemented by derive macro -impl AdminAuthEvent { - pub fn fire( - api: &ServerAPI, - payload: String, - target: Identifier, - db: Db, - output: Arc>, - ) { - api.event_bus.fire(&mut AdminAuthEvent { - cancelled: false, - id: ("core".to_string(), "admin_auth_event".to_string()), - payload, - target, - db, - output, - }); - } +// impl AdminAuthEvent { +// pub fn fire( +// api: &ServerAPI, +// payload: String, +// target: Identifier, +// db: Db, +// output: Arc>, +// ) { +// api.event_bus.fire(&mut AdminAuthEvent { +// cancelled: false, +// id: ("core".to_string(), "admin_auth_event".to_string()), +// payload, +// target, +// db, +// output, +// }); +// } - pub fn check(api: &ServerAPI, payload: String, target: Identifier, db: Db) -> bool { - let output = Arc::new(RwLock::new(false)); - Self::fire(api, payload, target, db, output.clone()); - *output.read().unwrap() - } -} +// pub fn check(api: &ServerAPI, payload: String, target: Identifier, db: Db) -> bool { +// let output = Arc::new(RwLock::new(false)); +// Self::fire(api, payload, target, db, output.clone()); +// *output.read().unwrap() +// } +// } -#[event_handler( - namespace = "core", - name = "admin_auth_event", - ctx = api.cfg.config_toml.cgrpc_token.clone() -)] -fn admin_auth_handler(event: &mut AdminAuthEvent, token: &Option) { - match token.as_deref() { - None => *event.output.write().unwrap() = true, - Some(token) if token == event.payload.as_str() => *event.output.write().unwrap() = true, - Some(_) => {} - } -} +// #[event_handler( +// namespace = "core", +// name = "admin_auth_event", +// ctx = api.cfg.config_toml.cgrpc_token.clone() +// )] +// fn admin_auth_handler(event: &mut AdminAuthEvent, token: &Option) { +// match token.as_deref() { +// None => *event.output.write().unwrap() = true, +// Some(token) if token == event.payload.as_str() => *event.output.write().unwrap() = true, +// Some(_) => {} +// } +// } diff --git a/enginelib/src/events/mod.rs b/enginelib/src/events/mod.rs index de30e31..0b9020f 100644 --- a/enginelib/src/events/mod.rs +++ b/enginelib/src/events/mod.rs @@ -36,9 +36,9 @@ impl Events { auth_event::AuthEvent::check(api, uid, challenge, db) } - pub fn CheckAdminAuth(api: &ServerAPI, payload: String, target: Identifier, db: Db) -> bool { - admin_auth_event::AdminAuthEvent::check(api, payload, target, db) - } + // pub fn CheckAdminAuth(api: &ServerAPI, payload: String, target: Identifier, db: Db) -> bool { + // admin_auth_event::AdminAuthEvent::check(api, payload, target, db) + // } pub fn CgrpcEvent( api: &ServerAPI, diff --git a/enginelib/src/prelude.rs b/enginelib/src/prelude.rs index 899d880..8e84693 100644 --- a/enginelib/src/prelude.rs +++ b/enginelib/src/prelude.rs @@ -10,7 +10,7 @@ pub use crate::event::{ Event, EventBus, EventCTX, EventHandler, EventRegistrar, register_inventory_handlers, register_inventory_handlers_for_origin, }; -pub use crate::events::admin_auth_event::AdminAuthEvent; +//pub use crate::events::admin_auth_event::AdminAuthEvent; pub use crate::events::auth_event::AuthEvent; pub use crate::events::before_task_block_acquire_event::BeforeTaskBlockAcquireEvent; pub use crate::events::before_task_block_execute_event::BeforeTaskBlockExecuteEvent; From b6e833f702c253c0ed0249bb2e9c3bee801f3af2 Mon Sep 17 00:00:00 2001 From: Styly Date: Sat, 18 Jul 2026 13:32:34 +0100 Subject: [PATCH 20/28] rewrite: scafold main.rs --- engine/src/bin/server.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/engine/src/bin/server.rs b/engine/src/bin/server.rs index f328e4d..00eb9d3 100644 --- a/engine/src/bin/server.rs +++ b/engine/src/bin/server.rs @@ -1 +1,7 @@ -fn main() {} +use enginelib::api::ServerAPI; + +fn main() { + let mut api = ServerAPI::default(); + let server = enginelib::server::RPC::new(&mut api); + server.run(); +} From 9d19337fac90e8e5c024ec5494a49e0c7cce2efc Mon Sep 17 00:00:00 2001 From: Styly Date: Sat, 18 Jul 2026 13:45:23 +0100 Subject: [PATCH 21/28] rewrite: finish scafold in main.rs --- enginelib/src/api.rs | 4 +++- enginelib/src/lib.rs | 1 + enginelib/src/nucleus/mod.rs | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 enginelib/src/nucleus/mod.rs diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index 2e6554d..7b083d1 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -14,6 +14,7 @@ pub use postcard::from_bytes; pub use postcard::to_allocvec; use std::{ collections::HashMap, + sync::RwLock, sync::{ Arc, atomic::{AtomicU64, Ordering}, @@ -22,7 +23,8 @@ use std::{ }; pub struct ServerAPI { - pub cfg: Config, // RW + pub cfg: RwLock, // RW pub event_bus: EventBus, // RW pub lib_manager: LibraryManager, // RW + pub db: sled::Db, } diff --git a/enginelib/src/lib.rs b/enginelib/src/lib.rs index 7fc70e0..9fd4744 100644 --- a/enginelib/src/lib.rs +++ b/enginelib/src/lib.rs @@ -3,6 +3,7 @@ pub mod api; pub mod config; pub mod event; pub mod events; +pub mod nucleus; extern crate self as enginelib; pub use inventory; pub mod plugin; diff --git a/enginelib/src/nucleus/mod.rs b/enginelib/src/nucleus/mod.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/enginelib/src/nucleus/mod.rs @@ -0,0 +1 @@ + From 66240056125c4b9bfaf7f6d4b9728418b00927a8 Mon Sep 17 00:00:00 2001 From: Styly Date: Sat, 18 Jul 2026 13:50:17 +0100 Subject: [PATCH 22/28] rewrite: nucleus scafold --- enginelib/src/nucleus/cancel.rs | 0 enginelib/src/nucleus/complete.rs | 0 enginelib/src/nucleus/lease.rs | 0 enginelib/src/nucleus/mod.rs | 7 ++++++- enginelib/src/nucleus/query.rs | 0 enginelib/src/nucleus/renew.rs | 0 enginelib/src/nucleus/submit.rs | 0 7 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 enginelib/src/nucleus/cancel.rs create mode 100644 enginelib/src/nucleus/complete.rs create mode 100644 enginelib/src/nucleus/lease.rs create mode 100644 enginelib/src/nucleus/query.rs create mode 100644 enginelib/src/nucleus/renew.rs create mode 100644 enginelib/src/nucleus/submit.rs diff --git a/enginelib/src/nucleus/cancel.rs b/enginelib/src/nucleus/cancel.rs new file mode 100644 index 0000000..e69de29 diff --git a/enginelib/src/nucleus/complete.rs b/enginelib/src/nucleus/complete.rs new file mode 100644 index 0000000..e69de29 diff --git a/enginelib/src/nucleus/lease.rs b/enginelib/src/nucleus/lease.rs new file mode 100644 index 0000000..e69de29 diff --git a/enginelib/src/nucleus/mod.rs b/enginelib/src/nucleus/mod.rs index 8b13789..600e8bb 100644 --- a/enginelib/src/nucleus/mod.rs +++ b/enginelib/src/nucleus/mod.rs @@ -1 +1,6 @@ - +mod cancel; +mod complete; +mod lease; +mod query; +mod renew; +mod submit; diff --git a/enginelib/src/nucleus/query.rs b/enginelib/src/nucleus/query.rs new file mode 100644 index 0000000..e69de29 diff --git a/enginelib/src/nucleus/renew.rs b/enginelib/src/nucleus/renew.rs new file mode 100644 index 0000000..e69de29 diff --git a/enginelib/src/nucleus/submit.rs b/enginelib/src/nucleus/submit.rs new file mode 100644 index 0000000..e69de29 From c3b005194c8a83839d21a29bc7444bd36be35893 Mon Sep 17 00:00:00 2001 From: Styly Date: Sat, 18 Jul 2026 16:23:58 +0100 Subject: [PATCH 23/28] rewrite: mig to rocksdb --- Cargo.lock | 346 ++++++++++++++++++----------- enginelib/Cargo.toml | 4 +- enginelib/src/api.rs | 17 +- enginelib/src/event.rs | 2 +- enginelib/src/events/auth_event.rs | 9 +- enginelib/src/events/mod.rs | 9 +- 6 files changed, 251 insertions(+), 136 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0aecd68..39e0712 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -69,10 +69,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] -name = "bitflags" -version = "1.3.2" +name = "bindgen" +version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", +] [[package]] name = "bitflags" @@ -118,6 +130,16 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "camino" version = "1.2.2" @@ -158,9 +180,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -181,6 +214,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading 0.8.9", +] + [[package]] name = "clru" version = "0.6.3" @@ -350,7 +394,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core 0.9.12", + "parking_lot_core", ] [[package]] @@ -430,6 +474,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + [[package]] name = "embedded-io" version = "0.4.0" @@ -468,12 +518,12 @@ dependencies = [ "dashmap", "directories", "inventory", - "libloading", + "libloading 0.9.0", "macros", "oxifs", "postcard", + "rust-rocksdb", "serde", - "sled", "tokio", "toml", "tracing", @@ -570,31 +620,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "futures-core" version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -674,7 +705,7 @@ dependencies = [ "gix-utils", "gix-validate", "gix-worktree", - "parking_lot 0.12.5", + "parking_lot", "signal-hook 0.3.18", "smallvec", "thiserror", @@ -781,7 +812,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2409cffa4fe8b303847d5b6ba8df9da9ba65d302fc5ee474ea0cac5afde79840" dependencies = [ - "bitflags 2.11.1", + "bitflags", "bstr", "gix-path", "libc", @@ -920,7 +951,7 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8546300aee4c65c5862c22a3e321124a69b654a61a8b60de546a9284812b7e2" dependencies = [ - "bitflags 2.11.1", + "bitflags", "bstr", "gix-features", "gix-path", @@ -946,7 +977,7 @@ checksum = "222f7428636020bef272a87ed833ea48bf5fb3193f99852ae16fbb5a602bd2f0" dependencies = [ "gix-hash", "hashbrown 0.16.1", - "parking_lot 0.12.5", + "parking_lot", ] [[package]] @@ -968,7 +999,7 @@ version = "0.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea6d3e9e11647ba49f441dea0782494cc6d2875ff43fa4ad9094e6957f42051" dependencies = [ - "bitflags 2.11.1", + "bitflags", "bstr", "filetime", "fnv", @@ -1038,7 +1069,7 @@ dependencies = [ "gix-pack", "gix-path", "gix-quote", - "parking_lot 0.12.5", + "parking_lot", "tempfile", "thiserror", ] @@ -1091,7 +1122,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed9e0c881933c37a7ef45288d6c5779c4a7b3ad240b4c37657e1d9829eb90085" dependencies = [ - "bitflags 2.11.1", + "bitflags", "bstr", "gix-attributes", "gix-config-value", @@ -1172,7 +1203,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91898c83b18c635696f7355d171cfa74a52f38022ff89581f567768935ebc4c8" dependencies = [ - "bitflags 2.11.1", + "bitflags", "bstr", "gix-commitgraph", "gix-date", @@ -1205,7 +1236,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea9962ed6d9114f7f100efe038752f41283c225bb507a2888903ac593dffa6be" dependencies = [ - "bitflags 2.11.1", + "bitflags", "gix-path", "libc", "windows-sys", @@ -1270,7 +1301,7 @@ dependencies = [ "dashmap", "gix-fs", "libc", - "parking_lot 0.12.5", + "parking_lot", "signal-hook 0.4.4", "signal-hook-registry", "tempfile", @@ -1304,7 +1335,7 @@ version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d052b83d1d1744be95ac6448ac02f95f370a8f6720e466be9ce57146e39f5280" dependencies = [ - "bitflags 2.11.1", + "bitflags", "gix-commitgraph", "gix-date", "gix-hash", @@ -1368,6 +1399,12 @@ dependencies = [ "gix-validate", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "hash32" version = "0.2.1" @@ -1506,21 +1543,21 @@ dependencies = [ ] [[package]] -name = "instant" -version = "0.1.13" +name = "inventory" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" dependencies = [ - "cfg-if", + "rustversion", ] [[package]] -name = "inventory" -version = "0.3.24" +name = "itertools" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" dependencies = [ - "rustversion", + "either", ] [[package]] @@ -1570,6 +1607,16 @@ dependencies = [ "jiff-tzdb", ] +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.2", + "libc", +] + [[package]] name = "js-sys" version = "0.3.95" @@ -1607,6 +1654,16 @@ version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libloading" version = "0.9.0" @@ -1623,12 +1680,23 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" dependencies = [ - "bitflags 2.11.1", + "bitflags", "libc", "plain", "redox_syscall 0.7.4", ] +[[package]] +name = "libz-sys" +version = "1.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1650,6 +1718,16 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "macros" version = "0.1.0" @@ -1694,6 +1772,12 @@ dependencies = [ "libc", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "mio" version = "1.2.0" @@ -1705,6 +1789,16 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1766,17 +1860,6 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] - [[package]] name = "parking_lot" version = "0.12.5" @@ -1784,21 +1867,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", - "parking_lot_core 0.9.12", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "winapi", + "parking_lot_core", ] [[package]] @@ -1826,6 +1895,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "plain" version = "0.2.3" @@ -1891,7 +1966,7 @@ version = "30.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a6efc566849d3d9d737c5cb06cc50e48950ebe3d3f9d70631490fff3a07b139" dependencies = [ - "parking_lot 0.12.5", + "parking_lot", ] [[package]] @@ -1909,22 +1984,13 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.1", + "bitflags", ] [[package]] @@ -1933,7 +1999,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" dependencies = [ - "bitflags 2.11.1", + "bitflags", ] [[package]] @@ -1976,6 +2042,42 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "rust-librocksdb-sys" +version = "0.47.0+11.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "040b883ae99f5f0e40bd93cd9ae7bdc9b2e53e58cc19a16a8cb503685f79b834" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "glob", + "libc", + "libz-sys", + "lz4-sys", + "pkg-config", + "rustflags", + "tikv-jemalloc-sys", + "zstd-sys", +] + +[[package]] +name = "rust-rocksdb" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a7b79a9e8015b4cd08caa31aae5208beffd4a851853df1abd30a8f35b311309" +dependencies = [ + "libc", + "parking_lot", + "rust-librocksdb-sys", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustc_version" version = "0.4.1" @@ -1985,13 +2087,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustflags" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a39e0e9135d7a7208ee80aa4e3e4b88f0f5ad7be92153ed70686c38a03db2e63" + [[package]] name = "rustix" version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.1", + "bitflags", "errno", "libc", "linux-raw-sys", @@ -2153,22 +2261,6 @@ dependencies = [ "libc", ] -[[package]] -name = "sled" -version = "0.34.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" -dependencies = [ - "crc32fast", - "crossbeam-epoch", - "crossbeam-utils", - "fs2", - "fxhash", - "libc", - "log", - "parking_lot 0.11.2", -] - [[package]] name = "smallvec" version = "1.15.1" @@ -2276,6 +2368,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "time" version = "0.3.47" @@ -2333,7 +2435,7 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot 0.12.5", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -2512,6 +2614,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "vergen" version = "9.1.0" @@ -2667,28 +2775,12 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.11.1", + "bitflags", "hashbrown 0.15.5", "indexmap", "semver", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - [[package]] name = "winapi-util" version = "0.1.11" @@ -2698,12 +2790,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-core" version = "0.62.2" @@ -2845,7 +2931,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.11.1", + "bitflags", "indexmap", "log", "serde", @@ -2896,3 +2982,13 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/enginelib/Cargo.toml b/enginelib/Cargo.toml index f1bbe62..1ca2863 100644 --- a/enginelib/Cargo.toml +++ b/enginelib/Cargo.toml @@ -16,7 +16,6 @@ toml = { workspace = true } tracing = "0.1.44" tracing-subscriber = "0.3.22" chrono = { version = "0.4.44", features = ["serde"] } -sled = "0.34.7" tokio = { version = "1.50.0", features = ["full"] } postcard = { version = "1.1.3", features = ["use-std"] } inventory = "0.3.22" @@ -29,3 +28,6 @@ vergen-gix = { version = "9.1.0", features = ["build", "cargo", "rustc"] } # tables in non-root member manifests. [dev-dependencies] tracing-test = "0.2.6" +[dependencies.rust-rocksdb] +features = ["jemalloc"] +version = "0.51.0" diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index 7b083d1..ea6bec9 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -26,5 +26,20 @@ pub struct ServerAPI { pub cfg: RwLock, // RW pub event_bus: EventBus, // RW pub lib_manager: LibraryManager, // RW - pub db: sled::Db, + pub db: rust_rocksdb::DB, +} +impl Default for ServerAPI { + fn default() -> Self { + let path = "./engine_db"; + let mut opts = rust_rocksdb::Options::default(); + opts.create_if_missing(true); + + let db = rust_rocksdb::DB::open(&opts, path).unwrap(); + Self { + cfg: RwLock::new(Config::default()), + event_bus: EventBus::default(), + lib_manager: LibraryManager::default(), + db: db, + } + } } diff --git a/enginelib/src/event.rs b/enginelib/src/event.rs index ab58f3b..98e0af6 100644 --- a/enginelib/src/event.rs +++ b/enginelib/src/event.rs @@ -53,7 +53,7 @@ pub trait EventCTX: EventHandler { fn handleCTX(&self, event: &mut C); } - +#[derive(Default, Clone)] pub struct EventBus { pub event_handler_registry: EngineEventHandlerRegistry, } diff --git a/enginelib/src/events/auth_event.rs b/enginelib/src/events/auth_event.rs index cd5fa0e..cbb0b7e 100644 --- a/enginelib/src/events/auth_event.rs +++ b/enginelib/src/events/auth_event.rs @@ -2,7 +2,6 @@ use std::sync::{Arc, RwLock}; use crate::{Identifier, api::ServerAPI}; use macros::{Event, event_handler}; -use sled::Db; #[derive(Clone, Debug, Event)] #[event(namespace = "core", name = "auth_event", cancellable)] @@ -11,7 +10,7 @@ pub struct AuthEvent { pub id: Identifier, pub uid: String, pub challenge: String, - pub db: Db, + pub db: Arc, pub output: Arc>, } impl AuthEvent { @@ -19,7 +18,7 @@ impl AuthEvent { api: &ServerAPI, uid: String, challenge: String, - db: Db, + db: rust_rocksdb::DB, output: Arc>, ) { api.event_bus.fire(&mut AuthEvent { @@ -27,12 +26,12 @@ impl AuthEvent { id: ("core".to_string(), "auth_event".to_string()), uid, challenge, - db, + db: Arc::new(db), output, }); } - pub fn check(api: &ServerAPI, uid: String, challenge: String, db: Db) -> bool { + pub fn check(api: &ServerAPI, uid: String, challenge: String, db: rust_rocksdb::DB) -> bool { let output = Arc::new(RwLock::new(false)); Self::fire(api, uid, challenge, db, output.clone()); *output.read().unwrap() diff --git a/enginelib/src/events/mod.rs b/enginelib/src/events/mod.rs index 0b9020f..ae15550 100644 --- a/enginelib/src/events/mod.rs +++ b/enginelib/src/events/mod.rs @@ -18,8 +18,6 @@ pub mod task_block_acquired_event; use std::collections::HashMap; use std::sync::{Arc, RwLock}; -use sled::Db; - use crate::{Identifier, api::ServerAPI}; pub fn ID(namespace: &str, id: &str) -> Identifier { @@ -32,7 +30,12 @@ pub struct Events; impl Events { pub fn init_auth(_api: &mut ServerAPI) {} - pub fn CheckAuth(api: &ServerAPI, uid: String, challenge: String, db: Db) -> bool { + pub fn CheckAuth( + api: &ServerAPI, + uid: String, + challenge: String, + db: rust_rocksdb::DB, + ) -> bool { auth_event::AuthEvent::check(api, uid, challenge, db) } From 6a5f959bb1c6186baa62537654db09bc809bdf69 Mon Sep 17 00:00:00 2001 From: Styly Date: Sat, 18 Jul 2026 17:35:48 +0100 Subject: [PATCH 24/28] rewrite: init --- enginelib/src/api.rs | 50 +++++++++++++++++++++++++++++++ enginelib/src/nucleus/cancel.rs | 5 ++++ enginelib/src/nucleus/complete.rs | 5 ++++ enginelib/src/nucleus/lease.rs | 5 ++++ enginelib/src/nucleus/query.rs | 5 ++++ enginelib/src/nucleus/renew.rs | 5 ++++ enginelib/src/nucleus/submit.rs | 12 ++++++++ 7 files changed, 87 insertions(+) diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index ea6bec9..e2dc3c9 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -3,6 +3,7 @@ use dashmap::{DashMap, DashSet}; use tokio::{spawn, time::interval}; use tracing::{Level, debug, info, instrument}; +use crate::Task; use crate::{ Identifier, Registry, config::Config, @@ -27,6 +28,9 @@ pub struct ServerAPI { pub event_bus: EventBus, // RW pub lib_manager: LibraryManager, // RW pub db: rust_rocksdb::DB, + // pub task_queue: TaskQueue, // RW + // pub leased_tasks: LeasedTaskQueue, // RW + pub task_registry: EngineTaskRegistry, // RW } impl Default for ServerAPI { fn default() -> Self { @@ -35,11 +39,57 @@ impl Default for ServerAPI { opts.create_if_missing(true); let db = rust_rocksdb::DB::open(&opts, path).unwrap(); + + use std::sync::OnceLock; + + static INIT: OnceLock<()> = OnceLock::new(); + INIT.get_or_init(|| { + #[cfg(debug_assertions)] + let _ = tracing_subscriber::FmtSubscriber::builder() + // all spans/events with a level higher than TRACE (e.g, debug, info, warn, etc.) + // will be written to stdout. + .with_max_level(Level::DEBUG) + // builds the subscriber. + .try_init(); + #[cfg(not(debug_assertions))] + let _ = tracing_subscriber::FmtSubscriber::builder() + // all spans/events with a level higher than TRACE (e.g, debug, info, warn, etc.) + // will be written to stdout. + .with_max_level(Level::INFO) + // builds the subscriber. + .try_init(); + }); + Self { cfg: RwLock::new(Config::default()), event_bus: EventBus::default(), lib_manager: LibraryManager::default(), db: db, + task_registry: EngineTaskRegistry::default(), } } } +impl ServerAPI { + pub fn init(api: &mut Self) { + api.lib_manager.load_modules(api); + } +} +#[derive(Default, Clone, Debug)] +pub struct EngineTaskRegistry { + pub tasks: HashMap>, +} +impl Registry for EngineTaskRegistry { + #[instrument] + fn register(&mut self, task: Arc, identifier: Identifier) { + // Insert the task into the hashmap with (mod_id, identifier) as the key + debug!( + "TaskRegistry: Registering task {}.{}", + identifier.0, identifier.1 + ); + self.tasks.insert(identifier, task); + } + + fn get(&self, identifier: &Identifier) -> Option> { + self.tasks.get(identifier).map(|obj| obj.clone_box()) + } +} diff --git a/enginelib/src/nucleus/cancel.rs b/enginelib/src/nucleus/cancel.rs index e69de29..c731a8e 100644 --- a/enginelib/src/nucleus/cancel.rs +++ b/enginelib/src/nucleus/cancel.rs @@ -0,0 +1,5 @@ +use std::sync::Arc; + +use crate::api::ServerAPI; + +fn cancel(api: Arc) {} diff --git a/enginelib/src/nucleus/complete.rs b/enginelib/src/nucleus/complete.rs index e69de29..be6c144 100644 --- a/enginelib/src/nucleus/complete.rs +++ b/enginelib/src/nucleus/complete.rs @@ -0,0 +1,5 @@ +use std::sync::Arc; + +use crate::api::ServerAPI; + +fn complete(api: Arc) {} diff --git a/enginelib/src/nucleus/lease.rs b/enginelib/src/nucleus/lease.rs index e69de29..c036050 100644 --- a/enginelib/src/nucleus/lease.rs +++ b/enginelib/src/nucleus/lease.rs @@ -0,0 +1,5 @@ +use std::sync::Arc; + +use crate::api::ServerAPI; + +fn lease(api: Arc) {} diff --git a/enginelib/src/nucleus/query.rs b/enginelib/src/nucleus/query.rs index e69de29..8e68e50 100644 --- a/enginelib/src/nucleus/query.rs +++ b/enginelib/src/nucleus/query.rs @@ -0,0 +1,5 @@ +use std::sync::Arc; + +use crate::api::ServerAPI; + +fn query(api: Arc) {} diff --git a/enginelib/src/nucleus/renew.rs b/enginelib/src/nucleus/renew.rs index e69de29..0161a7b 100644 --- a/enginelib/src/nucleus/renew.rs +++ b/enginelib/src/nucleus/renew.rs @@ -0,0 +1,5 @@ +use std::sync::Arc; + +use crate::api::ServerAPI; + +fn renew(api: Arc) {} diff --git a/enginelib/src/nucleus/submit.rs b/enginelib/src/nucleus/submit.rs index e69de29..a9b1779 100644 --- a/enginelib/src/nucleus/submit.rs +++ b/enginelib/src/nucleus/submit.rs @@ -0,0 +1,12 @@ +use std::sync::Arc; + +use crate::{Identifier, api::ServerAPI, task::Task}; +// t:namespace:task_name: -> Serialized Task Record +// f:namespace:task_name: -> Finished Task Record +pub fn submit(api: Arc, task_bytes: &[u8], task_id: Identifier) { + // deserialize + let task = Task::verify(task_bytes); + // Verify Task + // upload Task + let mut db = api.db.put("t", value) +} From 7d02c30bdb20bf1d1e6464adeba9d62aeeb5b435 Mon Sep 17 00:00:00 2001 From: Styly Date: Sat, 18 Jul 2026 17:37:37 +0100 Subject: [PATCH 25/28] rewrite: fmt --- enginelib/src/api.rs | 8 ++------ enginelib/src/nucleus/submit.rs | 4 ++-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index e2dc3c9..2b3fce7 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -3,7 +3,7 @@ use dashmap::{DashMap, DashSet}; use tokio::{spawn, time::interval}; use tracing::{Level, debug, info, instrument}; -use crate::Task; +use crate::task::Task; use crate::{ Identifier, Registry, config::Config, @@ -69,11 +69,7 @@ impl Default for ServerAPI { } } } -impl ServerAPI { - pub fn init(api: &mut Self) { - api.lib_manager.load_modules(api); - } -} + #[derive(Default, Clone, Debug)] pub struct EngineTaskRegistry { pub tasks: HashMap>, diff --git a/enginelib/src/nucleus/submit.rs b/enginelib/src/nucleus/submit.rs index a9b1779..3a2372f 100644 --- a/enginelib/src/nucleus/submit.rs +++ b/enginelib/src/nucleus/submit.rs @@ -5,8 +5,8 @@ use crate::{Identifier, api::ServerAPI, task::Task}; // f:namespace:task_name: -> Finished Task Record pub fn submit(api: Arc, task_bytes: &[u8], task_id: Identifier) { // deserialize - let task = Task::verify(task_bytes); + // Verify Task // upload Task - let mut db = api.db.put("t", value) + //let mut db = api.db.put("t", value) } From 901458073b73e7bdbedb41a375e1c6546558c2f5 Mon Sep 17 00:00:00 2001 From: Styly Date: Sat, 18 Jul 2026 17:44:42 +0100 Subject: [PATCH 26/28] rewrite: plugin --- enginelib/src/api.rs | 6 ++++-- enginelib/src/plugin.rs | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index 2b3fce7..fc9cf73 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -60,13 +60,15 @@ impl Default for ServerAPI { .try_init(); }); - Self { + let mut k = Self { cfg: RwLock::new(Config::default()), event_bus: EventBus::default(), lib_manager: LibraryManager::default(), db: db, task_registry: EngineTaskRegistry::default(), - } + }; + LibraryManager::load_modules(&mut k); + k } } diff --git a/enginelib/src/plugin.rs b/enginelib/src/plugin.rs index d9fe447..8847b3d 100644 --- a/enginelib/src/plugin.rs +++ b/enginelib/src/plugin.rs @@ -64,7 +64,8 @@ impl LibraryManager { drop(self); } - pub fn load_modules(&mut self, api: &mut ServerAPI) { + pub fn load_modules(api: &mut ServerAPI) { + let mut library_manager = LibraryManager::default(); let dir_path = "./mods"; let mut files: Vec = Vec::new(); @@ -92,8 +93,9 @@ impl LibraryManager { info!("Found {} module(s) to load", files.len()); for file in files { - self.load_module(&file, api); + library_manager.load_module(&file, api); } + api.lib_manager = library_manager; } pub fn load_module(&mut self, path: &str, api: &mut ServerAPI) { From 0d50e99ac074a470a015b4b7c7b0f5720e8696f6 Mon Sep 17 00:00:00 2001 From: Styly Date: Sat, 18 Jul 2026 17:48:06 +0100 Subject: [PATCH 27/28] rewrite: init events --- engine/src/bin/server.rs | 4 ++-- enginelib/src/api.rs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/engine/src/bin/server.rs b/engine/src/bin/server.rs index 00eb9d3..b5fd299 100644 --- a/engine/src/bin/server.rs +++ b/engine/src/bin/server.rs @@ -2,6 +2,6 @@ use enginelib::api::ServerAPI; fn main() { let mut api = ServerAPI::default(); - let server = enginelib::server::RPC::new(&mut api); - server.run(); + // let server = enginelib::server::RPC::new(&mut api); + // server.run(); } diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index fc9cf73..f9de1b7 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -68,6 +68,7 @@ impl Default for ServerAPI { task_registry: EngineTaskRegistry::default(), }; LibraryManager::load_modules(&mut k); + crate::event::register_inventory_handlers(&mut k); k } } From 6649265e63a35fdfe726f896f1137d252df55bcf Mon Sep 17 00:00:00 2001 From: Styly Date: Sat, 18 Jul 2026 18:34:29 +0100 Subject: [PATCH 28/28] rewrite: submit --- Cargo.lock | 61 ++++++++++++++++++++++++++++++++- enginelib/Cargo.toml | 1 + enginelib/src/error.rs | 58 +++++++++++++++++++++++++++++++ enginelib/src/lib.rs | 1 + enginelib/src/nucleus/submit.rs | 32 ++++++++++++++--- enginelib/src/task.rs | 2 +- 6 files changed, 148 insertions(+), 7 deletions(-) create mode 100644 enginelib/src/error.rs diff --git a/Cargo.lock b/Cargo.lock index 39e0712..c6c74fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -200,6 +200,17 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core", +] + [[package]] name = "chrono" version = "0.4.44" @@ -267,6 +278,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -468,6 +488,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "druid" +version = "0.1.0" +source = "git+https://github.com/voltaero/druid.git#dc2e5dcab0568079ffb4bad9a58e489c51c7fbad" +dependencies = [ + "rand", + "uuid", +] + [[package]] name = "dunce" version = "1.0.5" @@ -517,6 +546,7 @@ dependencies = [ "crossbeam", "dashmap", "directories", + "druid", "inventory", "libloading 0.9.0", "macros", @@ -656,6 +686,7 @@ dependencies = [ "cfg-if", "libc", "r-efi", + "rand_core", "wasip2", "wasip3", ] @@ -1984,6 +2015,23 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2196,7 +2244,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -2608,6 +2656,17 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" diff --git a/enginelib/Cargo.toml b/enginelib/Cargo.toml index 1ca2863..e2512fe 100644 --- a/enginelib/Cargo.toml +++ b/enginelib/Cargo.toml @@ -22,6 +22,7 @@ inventory = "0.3.22" crossbeam = "0.8.4" dashmap = "6.2.1" async-channel = "2.5.0" +druid = {git="https://github.com/voltaero/druid.git"} [build-dependencies] vergen-gix = { version = "9.1.0", features = ["build", "cargo", "rustc"] } # Profiles moved to the workspace root Cargo.toml — Cargo ignores profile diff --git a/enginelib/src/error.rs b/enginelib/src/error.rs new file mode 100644 index 0000000..6699220 --- /dev/null +++ b/enginelib/src/error.rs @@ -0,0 +1,58 @@ +use std::*; +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ErrorKind { + NotFound, + NotSupported, + InvalidArgument, + IOError, + Unknown, +} +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Error { + message: String, +} + +impl Error { + pub fn new(message: String) -> Error { + Error { message } + } + + pub fn into_string(self) -> String { + self.into() + } + + /// Parse corresponding [`ErrorKind`] from error message. + pub fn kind(&self) -> ErrorKind { + match self.message.split(':').next().unwrap_or("") { + "NotFound" => ErrorKind::NotFound, + "Not implemented" => ErrorKind::NotSupported, + "Invalid argument" => ErrorKind::InvalidArgument, + "IO error" => ErrorKind::IOError, + _ => ErrorKind::Unknown, + } + } +} + +impl AsRef for Error { + fn as_ref(&self) -> &str { + &self.message + } +} + +impl From for String { + fn from(e: Error) -> String { + e.message + } +} + +impl std::error::Error for Error { + fn description(&self) -> &str { + &self.message + } +} + +impl fmt::Display for Error { + fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> { + self.message.fmt(formatter) + } +} diff --git a/enginelib/src/lib.rs b/enginelib/src/lib.rs index 9fd4744..6134f59 100644 --- a/enginelib/src/lib.rs +++ b/enginelib/src/lib.rs @@ -18,3 +18,4 @@ pub trait Registry: Default + Clone { fn get(&self, identifier: &Identifier) -> Option>; } pub use chrono; +mod error; diff --git a/enginelib/src/nucleus/submit.rs b/enginelib/src/nucleus/submit.rs index 3a2372f..099a92a 100644 --- a/enginelib/src/nucleus/submit.rs +++ b/enginelib/src/nucleus/submit.rs @@ -1,12 +1,34 @@ use std::sync::Arc; -use crate::{Identifier, api::ServerAPI, task::Task}; +use crate::{ + Identifier, Registry, + api::ServerAPI, + error::{Error, ErrorKind}, + task::Task, +}; // t:namespace:task_name: -> Serialized Task Record // f:namespace:task_name: -> Finished Task Record -pub fn submit(api: Arc, task_bytes: &[u8], task_id: Identifier) { +pub fn submit(api: Arc, task_bytes: &[u8], task_id: Identifier) -> Result<(), Error> { // deserialize - // Verify Task - // upload Task - //let mut db = api.db.put("t", value) + let task = api + .task_registry + .get(&task_id) + .ok_or(Error::new("Not found".into()))?; + if task.verify(task_bytes) { + let res = api.db.put( + format!( + "t:{}:{}:{}", + task_id.0, + task_id.1, + druid::Druid::default().to_hex() + ), + task_bytes, + ); + if res.is_err() { + return Err(Error::new(res.err().unwrap().to_string())); + } + } + + Ok(()) } diff --git a/enginelib/src/task.rs b/enginelib/src/task.rs index 6978d8e..0798acf 100644 --- a/enginelib/src/task.rs +++ b/enginelib/src/task.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use tracing::{error, instrument, warn}; pub trait Verifiable { - fn verify(&self, b: Vec) -> bool; + fn verify(&self, b: &[u8]) -> bool; } pub trait Task: Debug + Sync + Send + Verifiable { fn get_id(&self) -> Identifier;