diff --git a/Cargo.lock b/Cargo.lock index 287b6d6..c6c74fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,56 +26,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anstream" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" - -[[package]] -name = "anstyle-parse" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys", -] - [[package]] name = "anyhow" version = "1.0.102" @@ -92,14 +42,15 @@ dependencies = [ ] [[package]] -name = "async-trait" -version = "0.1.89" +name = "async-channel" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" dependencies = [ - "proc-macro2", - "quote", - "syn", + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", ] [[package]] @@ -111,12 +62,6 @@ dependencies = [ "critical-section", ] -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "autocfg" version = "1.5.0" @@ -124,60 +69,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] -name = "axum" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" -dependencies = [ - "axum-core", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "serde_core", - "sync_wrapper", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.5.6" +name = "bindgen" +version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "sync_wrapper", - "tower-layer", - "tower-service", + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", ] -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.11.1" @@ -222,6 +130,16 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "camino" version = "1.2.2" @@ -262,9 +180,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -273,9 +202,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chacha20" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", "cpufeatures 0.3.0", @@ -297,54 +226,16 @@ dependencies = [ ] [[package]] -name = "clap" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", -] - -[[package]] -name = "clap_complete" -version = "4.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff7a1dccbdd8b078c2bdebff47e404615151534d5043da397ec50286816f9cb" -dependencies = [ - "clap", -] - -[[package]] -name = "clap_derive" -version = "4.6.0" +name = "clang-sys" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn", + "glob", + "libc", + "libloading 0.8.9", ] -[[package]] -name = "clap_lex" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" - [[package]] name = "clru" version = "0.6.3" @@ -364,18 +255,12 @@ dependencies = [ ] [[package]] -name = "colorchoice" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" - -[[package]] -name = "colored" -version = "3.1.1" +name = "concurrent-queue" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" dependencies = [ - "windows-sys", + "crossbeam-utils", ] [[package]] @@ -520,16 +405,16 @@ dependencies = [ [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ "cfg-if", "crossbeam-utils", "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core 0.9.12", + "parking_lot_core", ] [[package]] @@ -606,7 +491,7 @@ dependencies = [ [[package]] name = "druid" version = "0.1.0" -source = "git+https://github.com/GrandEngineering/druid.git#dc2e5dcab0568079ffb4bad9a58e489c51c7fbad" +source = "git+https://github.com/voltaero/druid.git#dc2e5dcab0568079ffb4bad9a58e489c51c7fbad" dependencies = [ "rand", "uuid", @@ -620,9 +505,9 @@ checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "embedded-io" @@ -649,36 +534,26 @@ dependencies = [ name = "engine" version = "0.1.0" dependencies = [ - "clap", - "clap_complete", - "colored", - "druid", "enginelib", - "prost", - "serde", - "tokio", - "toml", - "tonic", - "tonic-build", - "tonic-prost", - "tonic-prost-build", - "tonic-reflection", ] [[package]] name = "enginelib" version = "0.2.0" dependencies = [ + "async-channel", "chrono", "crossbeam", + "dashmap", "directories", + "druid", "inventory", - "libloading", + "libloading 0.9.0", "macros", "oxifs", "postcard", + "rust-rocksdb", "serde", - "sled", "tokio", "toml", "tracing", @@ -703,6 +578,27 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "faster-hex" version = "0.10.0" @@ -736,12 +632,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" -[[package]] -name = "fixedbitset" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" - [[package]] name = "fnv" version = "1.0.7" @@ -760,63 +650,11 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "futures-channel" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" -dependencies = [ - "futures-core", -] - [[package]] name = "futures-core" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" - -[[package]] -name = "futures-sink" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" - -[[package]] -name = "futures-task" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" - -[[package]] -name = "futures-util" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" -dependencies = [ - "futures-core", - "futures-task", - "pin-project-lite", - "slab", -] - -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "generic-array" @@ -898,7 +736,7 @@ dependencies = [ "gix-utils", "gix-validate", "gix-worktree", - "parking_lot 0.12.5", + "parking_lot", "signal-hook 0.3.18", "smallvec", "thiserror", @@ -1005,7 +843,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2409cffa4fe8b303847d5b6ba8df9da9ba65d302fc5ee474ea0cac5afde79840" dependencies = [ - "bitflags 2.11.1", + "bitflags", "bstr", "gix-path", "libc", @@ -1144,7 +982,7 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8546300aee4c65c5862c22a3e321124a69b654a61a8b60de546a9284812b7e2" dependencies = [ - "bitflags 2.11.1", + "bitflags", "bstr", "gix-features", "gix-path", @@ -1170,7 +1008,7 @@ checksum = "222f7428636020bef272a87ed833ea48bf5fb3193f99852ae16fbb5a602bd2f0" dependencies = [ "gix-hash", "hashbrown 0.16.1", - "parking_lot 0.12.5", + "parking_lot", ] [[package]] @@ -1192,7 +1030,7 @@ version = "0.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea6d3e9e11647ba49f441dea0782494cc6d2875ff43fa4ad9094e6957f42051" dependencies = [ - "bitflags 2.11.1", + "bitflags", "bstr", "filetime", "fnv", @@ -1262,7 +1100,7 @@ dependencies = [ "gix-pack", "gix-path", "gix-quote", - "parking_lot 0.12.5", + "parking_lot", "tempfile", "thiserror", ] @@ -1315,7 +1153,7 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed9e0c881933c37a7ef45288d6c5779c4a7b3ad240b4c37657e1d9829eb90085" dependencies = [ - "bitflags 2.11.1", + "bitflags", "bstr", "gix-attributes", "gix-config-value", @@ -1396,7 +1234,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91898c83b18c635696f7355d171cfa74a52f38022ff89581f567768935ebc4c8" dependencies = [ - "bitflags 2.11.1", + "bitflags", "bstr", "gix-commitgraph", "gix-date", @@ -1429,7 +1267,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea9962ed6d9114f7f100efe038752f41283c225bb507a2888903ac593dffa6be" dependencies = [ - "bitflags 2.11.1", + "bitflags", "gix-path", "libc", "windows-sys", @@ -1494,7 +1332,7 @@ dependencies = [ "dashmap", "gix-fs", "libc", - "parking_lot 0.12.5", + "parking_lot", "signal-hook 0.4.4", "signal-hook-registry", "tempfile", @@ -1528,7 +1366,7 @@ version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d052b83d1d1744be95ac6448ac02f95f370a8f6720e466be9ce57146e39f5280" dependencies = [ - "bitflags 2.11.1", + "bitflags", "gix-commitgraph", "gix-date", "gix-hash", @@ -1593,23 +1431,10 @@ dependencies = [ ] [[package]] -name = "h2" -version = "0.4.13" +name = "glob" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "hash32" @@ -1691,106 +1516,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "http" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" -dependencies = [ - "bytes", - "itoa", -] - -[[package]] -name = "http-body" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" -dependencies = [ - "bytes", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "hyper" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" -dependencies = [ - "atomic-waker", - "bytes", - "futures-channel", - "futures-core", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-timeout" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" -dependencies = [ - "hyper", - "hyper-util", - "pin-project-lite", - "tokio", - "tower-service", -] - -[[package]] -name = "hyper-util" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "http", - "http-body", - "hyper", - "libc", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", -] - [[package]] name = "iana-time-zone" version = "0.1.65" @@ -1848,15 +1573,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", -] - [[package]] name = "inventory" version = "0.3.24" @@ -1866,17 +1582,11 @@ dependencies = [ "rustversion", ] -[[package]] -name = "is_terminal_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" - [[package]] name = "itertools" -version = "0.14.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" dependencies = [ "either", ] @@ -1928,6 +1638,16 @@ dependencies = [ "jiff-tzdb", ] +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.2", + "libc", +] + [[package]] name = "js-sys" version = "0.3.95" @@ -1965,6 +1685,16 @@ version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libloading" version = "0.9.0" @@ -1981,12 +1711,23 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" dependencies = [ - "bitflags 2.11.1", + "bitflags", "libc", "plain", "redox_syscall 0.7.4", ] +[[package]] +name = "libz-sys" +version = "1.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2008,6 +1749,16 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "macros" version = "0.1.0" @@ -2026,12 +1777,6 @@ dependencies = [ "regex-automata", ] -[[package]] -name = "matchit" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" - [[package]] name = "maybe-async" version = "0.2.10" @@ -2059,10 +1804,10 @@ dependencies = [ ] [[package]] -name = "mime" -version = "0.3.17" +name = "minimal-lexical" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "mio" @@ -2076,10 +1821,14 @@ dependencies = [ ] [[package]] -name = "multimap" -version = "0.10.1" +name = "nom" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] [[package]] name = "nu-ansi-term" @@ -2120,12 +1869,6 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" - [[package]] name = "option-ext" version = "0.2.0" @@ -2143,15 +1886,10 @@ dependencies = [ ] [[package]] -name = "parking_lot" -version = "0.11.2" +name = "parking" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" @@ -2160,21 +1898,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", - "parking_lot_core 0.9.12", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "winapi", + "parking_lot_core", ] [[package]] @@ -2196,43 +1920,18 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "petgraph" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" -dependencies = [ - "fixedbitset", - "hashbrown 0.15.5", - "indexmap", -] - -[[package]] -name = "pin-project" -version = "1.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "pin-project-lite" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + [[package]] name = "plain" version = "0.2.3" @@ -2298,80 +1997,7 @@ version = "30.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a6efc566849d3d9d737c5cb06cc50e48950ebe3d3f9d70631490fff3a07b139" dependencies = [ - "parking_lot 0.12.5", -] - -[[package]] -name = "prost" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" -dependencies = [ - "heck", - "itertools", - "log", - "multimap", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "pulldown-cmark", - "pulldown-cmark-to-cmark", - "regex", - "syn", - "tempfile", -] - -[[package]] -name = "prost-derive" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "prost-types" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" -dependencies = [ - "prost", -] - -[[package]] -name = "pulldown-cmark" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c3a14896dfa883796f1cb410461aef38810ea05f2b2c33c5aded3649095fdad" -dependencies = [ - "bitflags 2.11.1", - "memchr", - "unicase", -] - -[[package]] -name = "pulldown-cmark-to-cmark" -version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" -dependencies = [ - "pulldown-cmark", + "parking_lot", ] [[package]] @@ -2391,9 +2017,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", "getrandom 0.4.2", @@ -2406,22 +2032,13 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.1", + "bitflags", ] [[package]] @@ -2430,7 +2047,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" dependencies = [ - "bitflags 2.11.1", + "bitflags", ] [[package]] @@ -2473,6 +2090,42 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "rust-librocksdb-sys" +version = "0.47.0+11.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "040b883ae99f5f0e40bd93cd9ae7bdc9b2e53e58cc19a16a8cb503685f79b834" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "glob", + "libc", + "libz-sys", + "lz4-sys", + "pkg-config", + "rustflags", + "tikv-jemalloc-sys", + "zstd-sys", +] + +[[package]] +name = "rust-rocksdb" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a7b79a9e8015b4cd08caa31aae5208beffd4a851853df1abd30a8f35b311309" +dependencies = [ + "libc", + "parking_lot", + "rust-librocksdb-sys", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustc_version" version = "0.4.1" @@ -2482,13 +2135,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustflags" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a39e0e9135d7a7208ee80aa4e3e4b88f0f5ad7be92153ed70686c38a03db2e63" + [[package]] name = "rustix" version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.1", + "bitflags", "errno", "libc", "linux-raw-sys", @@ -2650,28 +2309,6 @@ dependencies = [ "libc", ] -[[package]] -name = "slab" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" - -[[package]] -name = "sled" -version = "0.34.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935" -dependencies = [ - "crc32fast", - "crossbeam-epoch", - "crossbeam-utils", - "fs2", - "fxhash", - "libc", - "log", - "parking_lot 0.11.2", -] - [[package]] name = "smallvec" version = "1.15.1" @@ -2726,12 +2363,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" - [[package]] name = "tar" version = "0.4.45" @@ -2785,6 +2416,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "time" version = "0.3.47" @@ -2842,7 +2483,7 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot 0.12.5", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -2861,30 +2502,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tokio-stream" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - [[package]] name = "toml" version = "1.1.2+spec-1.1.0" @@ -2924,119 +2541,6 @@ version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" -[[package]] -name = "tonic" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" -dependencies = [ - "async-trait", - "axum", - "base64", - "bytes", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-timeout", - "hyper-util", - "percent-encoding", - "pin-project", - "socket2", - "sync_wrapper", - "tokio", - "tokio-stream", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tonic-build" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1882ac3bf5ef12877d7ed57aad87e75154c11931c2ba7e6cde5e22d63522c734" -dependencies = [ - "prettyplease", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tonic-prost" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" -dependencies = [ - "bytes", - "prost", - "tonic", -] - -[[package]] -name = "tonic-prost-build" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3144df636917574672e93d0f56d7edec49f90305749c668df5101751bb8f95a" -dependencies = [ - "prettyplease", - "proc-macro2", - "prost-build", - "prost-types", - "quote", - "syn", - "tempfile", - "tonic-build", -] - -[[package]] -name = "tonic-reflection" -version = "0.14.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf0685a51e6d02b502ba0764002e766b7f3042aed13d9234925b6ffbfa3fca7" -dependencies = [ - "prost", - "prost-types", - "tokio", - "tokio-stream", - "tonic", - "tonic-prost", -] - -[[package]] -name = "tower" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" -dependencies = [ - "futures-core", - "futures-util", - "indexmap", - "pin-project-lite", - "slab", - "sync_wrapper", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - [[package]] name = "tracing" version = "0.1.44" @@ -3119,24 +2623,12 @@ dependencies = [ "syn", ] -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - [[package]] name = "typenum" version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" -[[package]] -name = "unicase" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" - [[package]] name = "unicode-bom" version = "2.0.3" @@ -3164,17 +2656,11 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - [[package]] name = "uuid" -version = "1.23.0" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -3187,6 +2673,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "vergen" version = "9.1.0" @@ -3245,15 +2737,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -3351,28 +2834,12 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.11.1", + "bitflags", "hashbrown 0.15.5", "indexmap", "semver", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - [[package]] name = "winapi-util" version = "0.1.11" @@ -3382,12 +2849,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-core" version = "0.62.2" @@ -3529,7 +2990,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.11.1", + "bitflags", "indexmap", "log", "serde", @@ -3580,3 +3041,13 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/engine/Cargo.toml b/engine/Cargo.toml index 9e82df7..865e27a 100644 --- a/engine/Cargo.toml +++ b/engine/Cargo.toml @@ -9,23 +9,5 @@ description = "A Blazingly fast distributed task system" default = [] dev = [] [dependencies] -clap = { version = "4.5.60", features = ["derive"] } -clap_complete = "4.5.66" -colored = "3.1.1" -# directories = "5.0.1" -druid = { git = "https://github.com/GrandEngineering/druid.git" } -enginelib = { path = "../enginelib" } -# libloading = "0.8.6" -prost = "0.14" -serde = { workspace = true } -# serde = "1.0.219" -tokio = { version = "1.50.0", features = ["rt-multi-thread", "macros"] } -toml = { workspace = true } -# toml = "0.8.19" -tonic = "0.14" -tonic-prost = "0.14.5" -tonic-reflection = "0.14" -[build-dependencies] -tonic-build = "0.14" -tonic-prost-build = "0.14" +enginelib = { path = "../enginelib" } diff --git a/engine/build.rs b/engine/build.rs deleted file mode 100644 index 85f99ac..0000000 --- a/engine/build.rs +++ /dev/null @@ -1,11 +0,0 @@ -use std::error::Error; -use std::{env, path::PathBuf}; -fn main() -> Result<(), Box> { - let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); - - tonic_prost_build::configure() - .file_descriptor_set_path(out_dir.join("engine_descriptor.bin")) - .compile_protos(&["proto/engine.proto"], &["proto"])?; - - Ok(()) -} diff --git a/engine/proto/engine.proto b/engine/proto/engine.proto deleted file mode 100644 index 14a429d..0000000 --- a/engine/proto/engine.proto +++ /dev/null @@ -1,71 +0,0 @@ -syntax = "proto3"; -package engine; -service Engine { - rpc AquireTask(TaskRequest) returns (Task); - rpc AquireTaskReg(empty) returns (TaskRegistry); - rpc PublishTask(Task) returns (empty); - rpc cgrpc(cgrpcmsg) returns (cgrpcmsg); - rpc CreateTask(Task) returns (Task); - rpc DeleteTask(TaskSelector) returns (empty); - rpc GetTasks(TaskPageRequest) returns (TaskPage); - rpc CheckAuth(empty) returns (empty); - rpc GetMetadata(empty) returns (ServerMetadata); -} -message ServerMetadata { - string engine_api = 1; - repeated ModuleInfo mods = 2; -} -message ModuleInfo { - string mod_id = 1; - string api_version = 2; - string rustc_version = 3; - string mod_version = 4; -} -message TaskSelector { - TaskState state = 1; - string namespace = 2; - string task = 3; - string id = 4; -} -message empty {} -enum TaskState { - QUEUED = 0; - PROCESSING = 1; - SOLVED = 2; -} -message TaskPageRequest { - string namespace = 1; - string task = 2; - uint64 page = 3; - uint32 pageSize = 4; - TaskState state = 5; -} -message TaskPage { - string namespace = 1; - string task = 2; - uint64 page = 3; - uint32 pageSize = 4; - TaskState state = 5; - repeated Task tasks = 6; -} -message cgrpcmsg { - string handler_mod_id = 1; - string handler_id = 2; - bytes event_payload = 3; - string token = 4; -} - -message TaskRegistry { - repeated string tasks = 1; // namespace:task -} - -message TaskRequest { - string task_id = 1; // namespace:task - // bytes payload = 2; -} -message Task { - string id = 4; // the task unique identifier - bytes task_payload = 1; - string task_id = 2; // namespace:task - bytes payload = 3; -} diff --git a/engine/src/bin/client.rs b/engine/src/bin/client.rs deleted file mode 100644 index 546d42e..0000000 --- a/engine/src/bin/client.rs +++ /dev/null @@ -1,244 +0,0 @@ -use enginelib::{ - Registry, - api::EngineAPI, - events::Events, - event::info, - plugin::LibraryInstance, - prelude::debug, -}; -use proto::engine_client; -use std::{collections::HashMap, error::Error, sync::Arc}; -use tonic::{ - Request, - metadata::{MetadataKey, MetadataValue}, - transport::Endpoint, -}; - -pub mod proto { - tonic::include_proto!("engine"); -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - let mut api = EngineAPI::default_client(); - EngineAPI::init_client(&mut api); - Events::ClientStart(&api); - - compute_module(Arc::new(api)).await; - Ok(()) -} - -fn make_interceptor( - api_for_interceptor: Arc, -) -> impl FnMut(Request<()>) -> Result, tonic::Status> + Clone { - move |mut req: Request<()>| { - let headers = Arc::new(std::sync::RwLock::new(HashMap::::new())); - Events::ClientAuthPrepare(api_for_interceptor.as_ref(), headers.clone()); - - if let Ok(headers) = headers.read() { - for (key, value) in headers.iter() { - if let (Ok(key), Ok(value)) = ( - MetadataKey::from_bytes(key.as_bytes()), - MetadataValue::try_from(value.as_str()), - ) { - req.metadata_mut().insert(key, value); - } - } - } - - Ok(req) - } -} - -async fn worker_loop( - worker_id: usize, - api: Arc, - channel: tonic::transport::Channel, - task_ids: Arc>, -) { - let interceptor = make_interceptor(api.clone()); - let mut client = engine_client::EngineClient::with_interceptor(channel, interceptor); - - loop { - let mut got_any = false; - - for task_id in task_ids.iter() { - if Events::BeforeTaskAcquire(api.as_ref(), task_id.clone()) { - continue; - } - - let task_resp = client - .aquire_task(Request::new(proto::TaskRequest { - task_id: task_id.clone(), - })) - .await; - - let mut task_req = match task_resp { - Ok(resp) => { - got_any = true; - resp - } - Err(status) if status.code() == tonic::Code::NotFound => continue, - Err(status) if status.code() == tonic::Code::PermissionDenied => { - debug!( - "worker {}: auth failed during acquire for {}", - worker_id, task_id - ); - tokio::time::sleep(std::time::Duration::from_millis(200)).await; - continue; - } - Err(status) => { - debug!("worker {}: acquire failed for {}: {:?}", worker_id, task_id, status); - continue; - } - }; - - let task_payload = task_req.get_mut(); - - let acquired_payload = Arc::new(std::sync::RwLock::new(task_payload.task_payload.clone())); - Events::TaskAcquired( - api.as_ref(), - task_id.clone(), - task_payload.id.clone(), - acquired_payload.clone(), - ); - if let Ok(payload) = acquired_payload.read() { - task_payload.task_payload = payload.clone(); - } - - let exec_payload = Arc::new(std::sync::RwLock::new(task_payload.task_payload.clone())); - if Events::BeforeTaskExecute( - api.as_ref(), - task_id.clone(), - task_payload.id.clone(), - exec_payload.clone(), - ) { - continue; - } - if let Ok(payload) = exec_payload.read() { - task_payload.task_payload = payload.clone(); - } - - let identifier = match task_id.split_once(':') { - Some(v) => v, - None => continue, - }; - let task = match api - .task_registry - .get(&(identifier.0.to_string(), identifier.1.to_string())) - { - Some(t) => t, - None => continue, - }; - let mut task = task.from_bytes(&task_payload.task_payload); - - task.run_hip(); - - let mut solv_task = proto::Task { - id: task_payload.id.clone(), - payload: Vec::new(), - task_id: task_id.clone(), - task_payload: task.to_bytes(), - }; - - let publish_payload = Arc::new(std::sync::RwLock::new(solv_task.task_payload.clone())); - if Events::BeforeTaskPublish( - api.as_ref(), - task_id.clone(), - task_payload.id.clone(), - publish_payload.clone(), - ) { - continue; - } - if let Ok(payload) = publish_payload.read() { - solv_task.task_payload = payload.clone(); - } - - if let Err(status) = client.publish_task(Request::new(solv_task)).await { - debug!( - "worker {}: publish failed for {}: {:?}", - worker_id, task_id, status - ); - } - } - - if !got_any { - tokio::time::sleep(std::time::Duration::from_millis(2)).await; - } - } -} - -// Compute Module -// Verifies server and also is -// Responsible for getting task, executing and publishing it. -async fn compute_module(api: Arc) { - let url = "http://[::1]:50051"; - let endpoint = Endpoint::from_static(url) - .tcp_nodelay(true) - .http2_adaptive_window(true) - .keep_alive_while_idle(true) - .tcp_keepalive(Some(std::time::Duration::from_secs(30))); - - let channel = endpoint.connect().await.unwrap(); - let interceptor = make_interceptor(api.clone()); - let mut client = engine_client::EngineClient::with_interceptor(channel.clone(), interceptor); - - // Get server metadata - let server_meta = client - .get_metadata(Request::new(proto::Empty {})) - .await - .unwrap() - .into_inner(); - - // validate server - assert!(server_meta.engine_api == enginelib::GIT_VERSION); - for x in &server_meta.mods { - assert!(x.api_version == enginelib::GIT_VERSION); - #[cfg(not(debug_assertions))] - assert!(x.rustc_version == enginelib::RUSTC_VERSION); - assert!(api.lib_manager.libraries.contains_key(&x.mod_id)); - let module: &LibraryInstance = api - .lib_manager - .libraries - .get(&x.mod_id) - .expect("Client Missing Mod"); - assert!(module.metadata.mod_version == x.mod_version) - } - - let task_reg = client - .aquire_task_reg(Request::new(proto::Empty {})) - .await - .unwrap() - .into_inner(); - - let task_ids = Arc::new(task_reg.tasks); - let worker_count = std::env::var("GE_CLIENT_WORKERS") - .ok() - .and_then(|v| v.parse::().ok()) - .filter(|v| *v > 0) - .unwrap_or_else(|| { - std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(4) - }); - - info!( - "Starting {} client workers for {} task types", - worker_count, - task_ids.len() - ); - - let mut handles = Vec::with_capacity(worker_count); - for worker_id in 0..worker_count { - let api_i = api.clone(); - let channel_i = channel.clone(); - let task_ids_i = task_ids.clone(); - handles.push(tokio::spawn(async move { - worker_loop(worker_id, api_i, channel_i, task_ids_i).await; - })); - } - - for handle in handles { - let _ = handle.await; - } -} diff --git a/engine/src/bin/packer.rs b/engine/src/bin/packer.rs deleted file mode 100644 index 46cf05c..0000000 --- a/engine/src/bin/packer.rs +++ /dev/null @@ -1,1057 +0,0 @@ -use clap::{Args, CommandFactory, Subcommand, ValueEnum, ValueHint}; -use clap::{Command, Parser}; -use clap_complete::{Generator, Shell, generate}; -use colored::*; -use enginelib::events::{Events, ID}; -// For coloring the output -use enginelib::Registry; -use enginelib::api::postcard; -use enginelib::prelude::error; -use enginelib::task::{StoredTask, Task, TaskQueue}; -use enginelib::{api::EngineAPI, config::Config, event::info}; -use serde::Deserialize; -use std::collections::BTreeMap; -use std::collections::HashMap; -use std::ffi::OsString; -use std::fs::File; -use std::io::Write; -use std::io::{self, BufReader, ErrorKind, Read}; -use std::path::PathBuf; -use toml::Value; -use tonic::{ - Request, - metadata::{MetadataKey, MetadataValue}, - transport::Endpoint, -}; - -pub mod proto { - tonic::include_proto!("engine"); -} - -#[derive(Debug)] -struct Entry { - namespace: String, - id: String, - data: BTreeMap, -} - -#[derive(Debug, Deserialize)] -#[serde(transparent)] -struct RawDoc( - std::collections::BTreeMap>>, -); - -fn parse_entries(raw: RawDoc) -> Vec { - let mut result = Vec::new(); - - for (compound_key, records) in raw.0 { - // split on colon: "widget:button" -> ("widget", "button") - let mut parts = compound_key.splitn(2, ':'); - let namespace = parts.next().unwrap_or("").to_string(); - let id = parts.next().unwrap_or("").to_string(); - - for data in records { - result.push(Entry { - namespace: namespace.clone(), - id: id.clone(), - data, - }); - } - } - - result -} - -/// A simple CLI application -#[derive(Parser, Debug)] -#[command(name = "packer")] -#[command(version = "1.0")] -#[command(author = "GrandEngineering")] -#[command(about = "A simple CLI app to pack tasks")] -struct Cli { - #[command(subcommand)] - command: Option, - #[arg(long = "generate", value_enum)] - generator: Option, -} - -#[derive(Subcommand, Debug, PartialEq)] -enum Commands { - #[command()] - Pack(PackArgs), - #[command()] - Unpack(PackArgs), - #[command()] - Upload(PackArgs), - #[command()] - AdminCheck, - #[command()] - AdminList(ListArgs), - #[command()] - AdminExport(ExportArgs), - #[command()] - AdminDelete(DeleteArgs), - #[command()] - Schema, -} - -#[derive(Args, Debug, PartialEq)] -struct PackArgs { - #[arg(short, required = true, value_hint = ValueHint::FilePath)] - input: PathBuf, - #[arg(long)] - stream: bool, -} - -#[derive(Clone, Debug, ValueEnum, PartialEq)] -enum StateArg { - Queued, - Processing, - Solved, -} - -impl StateArg { - fn to_proto(&self) -> i32 { - match self { - StateArg::Queued => proto::TaskState::Queued as i32, - StateArg::Processing => proto::TaskState::Processing as i32, - StateArg::Solved => proto::TaskState::Solved as i32, - } - } - - fn as_str(&self) -> &'static str { - match self { - StateArg::Queued => "queued", - StateArg::Processing => "processing", - StateArg::Solved => "solved", - } - } -} - -#[derive(Args, Debug, PartialEq)] -struct ListArgs { - #[arg(long)] - task_id: Option, // namespace:task - #[arg(long, value_enum, default_value = "queued")] - state: StateArg, - #[arg(long)] - all_states: bool, - #[arg(long, default_value_t = 1000)] - page_size: u32, -} - -#[derive(Args, Debug, PartialEq)] -struct ExportArgs { - #[arg(short = 'o', long, value_hint = ValueHint::FilePath, default_value = "output.rustforge.bin")] - output: PathBuf, - #[arg(long)] - task_id: Option, // namespace:task - #[arg(long, value_enum, default_value = "queued")] - state: StateArg, - #[arg(long)] - all_states: bool, - #[arg(long, default_value_t = 1000)] - page_size: u32, -} - -#[derive(Args, Debug, PartialEq)] -struct DeleteArgs { - #[arg(long)] - namespace: String, - #[arg(long)] - task: String, - #[arg(long)] - id: String, - #[arg(long, value_enum)] - state: StateArg, -} -fn print_completions(generator: G, cmd: &mut Command) { - generate( - generator, - cmd, - cmd.get_name().to_string(), - &mut io::stdout(), - ); -} - -fn build_headers(api: &EngineAPI, admin: bool) -> HashMap { - let headers = std::sync::Arc::new(std::sync::RwLock::new(HashMap::::new())); - Events::ClientAuthPrepare(api, headers.clone()); - let mut prepared_headers = headers.read().map(|h| h.clone()).unwrap_or_default(); - - if admin { - if let Some(token) = api.cfg.config_toml.cgrpc_token.clone() { - prepared_headers.insert("authorization".to_string(), token); - } - } - - prepared_headers -} - -const CHUNKED_MAGIC: [u8; 4] = *b"RFPK"; -const CHUNKED_VERSION: u8 = 2; -const FRAME_TASK: u8 = 1; -const FRAME_END: u8 = 255; - -#[derive(Debug)] -struct ChunkedTaskRecord { - namespace: String, - task: String, - id: String, - payload: Vec, -} - -fn write_chunked_header(writer: &mut W) -> io::Result<()> { - writer.write_all(&CHUNKED_MAGIC)?; - writer.write_all(&[CHUNKED_VERSION])?; - Ok(()) -} - -fn write_chunked_task_record( - writer: &mut W, - record: &ChunkedTaskRecord, -) -> io::Result<()> { - let ns = record.namespace.as_bytes(); - let task = record.task.as_bytes(); - let id = record.id.as_bytes(); - - let ns_len: u16 = ns - .len() - .try_into() - .map_err(|_| io::Error::new(ErrorKind::InvalidInput, "namespace too large"))?; - let task_len: u16 = task - .len() - .try_into() - .map_err(|_| io::Error::new(ErrorKind::InvalidInput, "task too large"))?; - let id_len: u32 = id - .len() - .try_into() - .map_err(|_| io::Error::new(ErrorKind::InvalidInput, "id too large"))?; - let payload_len: u64 = record - .payload - .len() - .try_into() - .map_err(|_| io::Error::new(ErrorKind::InvalidInput, "payload too large"))?; - - let mut meta = [0u8; 1 + 2 + 2 + 4 + 8]; - meta[0] = FRAME_TASK; - meta[1..3].copy_from_slice(&ns_len.to_le_bytes()); - meta[3..5].copy_from_slice(&task_len.to_le_bytes()); - meta[5..9].copy_from_slice(&id_len.to_le_bytes()); - meta[9..17].copy_from_slice(&payload_len.to_le_bytes()); - - writer.write_all(&meta)?; - writer.write_all(ns)?; - writer.write_all(task)?; - writer.write_all(id)?; - writer.write_all(&record.payload)?; - - Ok(()) -} - -fn write_chunked_end(writer: &mut W) -> io::Result<()> { - writer.write_all(&[FRAME_END])?; - Ok(()) -} - -fn read_exact_or_eof(reader: &mut R, buf: &mut [u8]) -> io::Result { - match reader.read_exact(buf) { - Ok(()) => Ok(true), - Err(e) if e.kind() == ErrorKind::UnexpectedEof => Ok(false), - Err(e) => Err(e), - } -} - -fn read_chunked_header(reader: &mut R) -> io::Result<()> { - let mut magic = [0u8; 4]; - reader.read_exact(&mut magic)?; - if magic != CHUNKED_MAGIC { - return Err(io::Error::new( - ErrorKind::InvalidData, - "invalid chunked magic", - )); - } - - let mut version = [0u8; 1]; - reader.read_exact(&mut version)?; - if version[0] != CHUNKED_VERSION { - return Err(io::Error::new( - ErrorKind::InvalidData, - format!("unsupported chunked version {}", version[0]), - )); - } - - Ok(()) -} - -fn read_chunked_next(reader: &mut R) -> io::Result> { - let mut frame = [0u8; 1]; - if !read_exact_or_eof(reader, &mut frame)? { - return Ok(None); - } - - match frame[0] { - FRAME_END => Ok(None), - FRAME_TASK => { - let mut ns_len = [0u8; 2]; - let mut task_len = [0u8; 2]; - let mut id_len = [0u8; 4]; - let mut payload_len = [0u8; 8]; - reader.read_exact(&mut ns_len)?; - reader.read_exact(&mut task_len)?; - reader.read_exact(&mut id_len)?; - reader.read_exact(&mut payload_len)?; - - let ns_len = u16::from_le_bytes(ns_len) as usize; - let task_len = u16::from_le_bytes(task_len) as usize; - let id_len = u32::from_le_bytes(id_len) as usize; - let payload_len_u64 = u64::from_le_bytes(payload_len); - if payload_len_u64 > usize::MAX as u64 { - return Err(io::Error::new( - ErrorKind::InvalidData, - "payload length too large for this platform", - )); - } - let payload_len = payload_len_u64 as usize; - - let mut ns = vec![0u8; ns_len]; - let mut task = vec![0u8; task_len]; - let mut id = vec![0u8; id_len]; - let mut payload = vec![0u8; payload_len]; - - reader.read_exact(&mut ns)?; - reader.read_exact(&mut task)?; - reader.read_exact(&mut id)?; - reader.read_exact(&mut payload)?; - - let namespace = String::from_utf8(ns) - .map_err(|_| io::Error::new(ErrorKind::InvalidData, "invalid namespace utf8"))?; - let task = String::from_utf8(task) - .map_err(|_| io::Error::new(ErrorKind::InvalidData, "invalid task utf8"))?; - let id = String::from_utf8(id) - .map_err(|_| io::Error::new(ErrorKind::InvalidData, "invalid id utf8"))?; - - Ok(Some(ChunkedTaskRecord { - namespace, - task, - id, - payload, - })) - } - other => Err(io::Error::new( - ErrorKind::InvalidData, - format!("unknown frame type {}", other), - )), - } -} - -#[tokio::main] -async fn main() { - let cli = Cli::parse(); - if let Some(generator) = cli.generator { - let mut cmd = Cli::command(); - eprintln!("Generating completion file for {generator:?}..."); - print_completions(generator, &mut cmd); - } - let mut api = EngineAPI::default_client(); - EngineAPI::init_client(&mut api); - // packer intentionally uses client init path; load config explicitly for host/admin token - api.cfg = Config::new(); - for (id, tsk) in api.task_registry.tasks.iter() { - api.task_queue.tasks.entry(id.clone()).or_default(); - } - if let Some(command) = cli.command { - match command { - Commands::Schema => { - let mut buf: Vec = Vec::new(); - for tsk in api.task_registry.tasks { - let unw = tsk.1.to_toml(); - buf.push(format![r#"[["{}:{}"]]"#, tsk.0.0, tsk.0.1]); - buf.push(unw); - } - let ns = buf.join("\n"); - match File::create("schema.rustforge.toml") { - Ok(mut file) => { - if let Err(e) = file.write_all(ns.as_bytes()) { - error!("Failed to write schema file: {}", e); - } else { - info!("Wrote schema.rustforge.toml"); - } - } - Err(e) => { - error!("Failed to create schema file: {}", e); - } - } - } - Commands::Unpack(input) => { - if input.input.exists() { - info!("Unpacking File: {}", input.input.to_string_lossy()); - let mut buf = Vec::new(); - - // Attempt to open and read the input file. If either step fails, - // we do not proceed to deserialization or writing the output file. - match File::open(&input.input) { - Ok(mut f) => { - if let Err(e) = f.read_to_end(&mut buf) { - error!( - "Failed to read input file {}: {}", - input.input.display(), - e - ); - // reading failed -> do not proceed to deserialize or write - return; - } - } - Err(e) => { - error!("Failed to open input file {}: {}", input.input.display(), e); - // opening failed -> do not proceed to deserialize or write - return; - } - } - - // Try to deserialize. Only on successful deserialization do we - // process entries and write the output TOML file. - let maybe_queue: Option = - match postcard::from_bytes::(&buf) { - Ok(k) => Some(k), - Err(e) => { - error!("Failed to deserialize task queue: {}", e); - None - } - }; - - if let Some(k) = maybe_queue { - let mut final_out: Vec = Vec::new(); - - for tasks in k.tasks { - match api.task_registry.tasks.get(&tasks.0.clone()) { - Some(tt) => { - for task in tasks.1 { - if tt.verify(task.bytes.clone()) { - let tmp_nt = tt.from_bytes(&task.bytes); - final_out.push(format![ - r#"[["{}:{}"]]"#, - tasks.0.0.clone(), - tasks.0.1.clone() - ]); - final_out.push(tmp_nt.to_toml()); - info!("{:?}", tmp_nt); - } - } - } - None => { - error!("Unknown template for {}:{}", tasks.0.0, tasks.0.1); - } - } - } - - let ns = final_out.join("\n"); - match File::create("output.rustforge.toml") { - Ok(mut file) => { - if let Err(e) = file.write_all(ns.as_bytes()) { - error!("Failed to write output.rustforge.toml: {}", e); - } else { - info!("Wrote output.rustforge.toml"); - } - } - Err(e) => { - error!("Failed to create output.rustforge.toml: {}", e); - } - } - } else { - // Deserialization failed; we logged the error above and intentionally do not - // create/write the output file to avoid producing an empty output. - } - } - } - Commands::Upload(input) => { - if !input.input.exists() { - error!("File does not exist: {}", input.input.to_string_lossy()); - return; - } - - info!( - "Uploading File: {}{}", - input.input.to_string_lossy(), - if input.stream { " (stream mode)" } else { "" } - ); - - let prepared_headers = build_headers(&api, false); - - let interceptor = move |mut req: Request<()>| { - for (key, value) in prepared_headers.iter() { - if let (Ok(key), Ok(value)) = ( - MetadataKey::from_bytes(key.as_bytes()), - MetadataValue::try_from(value.as_str()), - ) { - req.metadata_mut().insert(key, value); - } - } - Ok(req) - }; - - let endpoint = format!("http://{}", api.cfg.config_toml.host); - let channel = match Endpoint::from_shared(endpoint.clone()) { - Ok(ep) => match ep.connect().await { - Ok(ch) => ch, - Err(e) => { - error!("Failed to connect to server {}: {}", endpoint, e); - return; - } - }, - Err(e) => { - error!("Invalid server endpoint {}: {}", endpoint, e); - return; - } - }; - - let mut client = - proto::engine_client::EngineClient::with_interceptor(channel, interceptor); - - let mut uploaded = 0usize; - let mut failed = 0usize; - - if input.stream { - let file = match File::open(&input.input) { - Ok(f) => f, - Err(e) => { - error!("Failed to open input file {}: {}", input.input.display(), e); - return; - } - }; - let mut reader = BufReader::new(file); - if let Err(e) = read_chunked_header(&mut reader) { - error!("Failed to read chunked stream header: {}", e); - return; - } - - loop { - let record = match read_chunked_next(&mut reader) { - Ok(v) => v, - Err(e) => { - error!("Failed to read chunked stream record: {}", e); - return; - } - }; - - let Some(record) = record else { - break; - }; - - let task_id = format!("{}:{}", record.namespace, record.task); - let req = proto::Task { - id: record.id, - task_id: task_id.clone(), - task_payload: record.payload, - payload: Vec::new(), - }; - - match client.create_task(Request::new(req)).await { - Ok(_) => uploaded += 1, - Err(e) => { - failed += 1; - error!("Failed to upload task {}: {}", task_id, e); - } - } - } - } else { - let mut buf = Vec::new(); - match File::open(&input.input) { - Ok(mut f) => { - if let Err(e) = f.read_to_end(&mut buf) { - error!( - "Failed to read input file {}: {}", - input.input.display(), - e - ); - return; - } - } - Err(e) => { - error!("Failed to open input file {}: {}", input.input.display(), e); - return; - } - } - - let queue: TaskQueue = match postcard::from_bytes::(&buf) { - Ok(q) => q, - Err(e) => { - error!("Failed to deserialize task queue: {}", e); - return; - } - }; - - for ((namespace, task), tasks) in queue.tasks { - let task_id = format!("{}:{}", namespace, task); - for stored in tasks { - let req = proto::Task { - id: stored.id, - task_id: task_id.clone(), - task_payload: stored.bytes, - payload: Vec::new(), - }; - match client.create_task(Request::new(req)).await { - Ok(_) => uploaded += 1, - Err(e) => { - failed += 1; - error!("Failed to upload task {}: {}", task_id, e); - } - } - } - } - } - - info!( - "Upload complete. uploaded={}, failed={}, mode={}", - uploaded, - failed, - if input.stream { "stream" } else { "legacy" } - ); - } - Commands::AdminCheck => { - let prepared_headers = build_headers(&api, true); - let interceptor = move |mut req: Request<()>| { - for (key, value) in prepared_headers.iter() { - if let (Ok(key), Ok(value)) = ( - MetadataKey::from_bytes(key.as_bytes()), - MetadataValue::try_from(value.as_str()), - ) { - req.metadata_mut().insert(key, value); - } - } - Ok(req) - }; - - let endpoint = format!("http://{}", api.cfg.config_toml.host); - let channel = match Endpoint::from_shared(endpoint.clone()) { - Ok(ep) => match ep.connect().await { - Ok(ch) => ch, - Err(e) => { - error!("Failed to connect to server {}: {}", endpoint, e); - return; - } - }, - Err(e) => { - error!("Invalid server endpoint {}: {}", endpoint, e); - return; - } - }; - - let mut client = - proto::engine_client::EngineClient::with_interceptor(channel, interceptor); - match client.check_auth(Request::new(proto::Empty {})).await { - Ok(_) => info!("Admin auth check: OK"), - Err(e) => error!("Admin auth check failed: {}", e), - } - } - Commands::AdminList(args) => { - let prepared_headers = build_headers(&api, true); - let interceptor = move |mut req: Request<()>| { - for (key, value) in prepared_headers.iter() { - if let (Ok(key), Ok(value)) = ( - MetadataKey::from_bytes(key.as_bytes()), - MetadataValue::try_from(value.as_str()), - ) { - req.metadata_mut().insert(key, value); - } - } - Ok(req) - }; - - let endpoint = format!("http://{}", api.cfg.config_toml.host); - let channel = match Endpoint::from_shared(endpoint.clone()) { - Ok(ep) => match ep.connect().await { - Ok(ch) => ch, - Err(e) => { - error!("Failed to connect to server {}: {}", endpoint, e); - return; - } - }, - Err(e) => { - error!("Invalid server endpoint {}: {}", endpoint, e); - return; - } - }; - - let mut client = - proto::engine_client::EngineClient::with_interceptor(channel, interceptor); - - let task_ids: Vec = if let Some(task_id) = args.task_id.clone() { - vec![task_id] - } else { - match client.aquire_task_reg(Request::new(proto::Empty {})).await { - Ok(res) => res.into_inner().tasks, - Err(e) => { - error!("Failed to fetch task registry: {}", e); - return; - } - } - }; - - let states: Vec = if args.all_states { - vec![StateArg::Queued, StateArg::Processing, StateArg::Solved] - } else { - vec![args.state.clone()] - }; - - let mut listed = 0usize; - println!("state\ttask\tid"); - for task_id in task_ids { - let Some((namespace, task)) = task_id.split_once(':') else { - error!("Invalid task id '{}' (expected namespace:task)", task_id); - continue; - }; - - for state in &states { - let mut page = 0u64; - loop { - let req = proto::TaskPageRequest { - namespace: namespace.to_string(), - task: task.to_string(), - page, - page_size: args.page_size, - state: state.to_proto(), - }; - - let resp = match client.get_tasks(Request::new(req)).await { - Ok(r) => r.into_inner(), - Err(e) => { - error!( - "GetTasks failed for {} state {:?}: {}", - task_id, state, e - ); - break; - } - }; - - if resp.tasks.is_empty() { - break; - } - - for t in resp.tasks { - println!("{}\t{}:{}\t{}", state.as_str(), namespace, task, t.id); - listed += 1; - } - - page += 1; - } - } - } - - info!("Listed {} task(s)", listed); - } - Commands::AdminExport(args) => { - let prepared_headers = build_headers(&api, true); - let interceptor = move |mut req: Request<()>| { - for (key, value) in prepared_headers.iter() { - if let (Ok(key), Ok(value)) = ( - MetadataKey::from_bytes(key.as_bytes()), - MetadataValue::try_from(value.as_str()), - ) { - req.metadata_mut().insert(key, value); - } - } - Ok(req) - }; - - let endpoint = format!("http://{}", api.cfg.config_toml.host); - let channel = match Endpoint::from_shared(endpoint.clone()) { - Ok(ep) => match ep.connect().await { - Ok(ch) => ch, - Err(e) => { - error!("Failed to connect to server {}: {}", endpoint, e); - return; - } - }, - Err(e) => { - error!("Invalid server endpoint {}: {}", endpoint, e); - return; - } - }; - - let mut client = - proto::engine_client::EngineClient::with_interceptor(channel, interceptor); - - let task_ids: Vec = if let Some(task_id) = args.task_id.clone() { - vec![task_id] - } else { - match client.aquire_task_reg(Request::new(proto::Empty {})).await { - Ok(res) => res.into_inner().tasks, - Err(e) => { - error!("Failed to fetch task registry: {}", e); - return; - } - } - }; - - let states: Vec = if args.all_states { - vec![StateArg::Queued, StateArg::Processing, StateArg::Solved] - } else { - vec![args.state.clone()] - }; - - let mut out_queue = TaskQueue::default(); - let mut fetched = 0usize; - - for task_id in task_ids { - let Some((namespace, task)) = task_id.split_once(':') else { - error!("Invalid task id '{}' (expected namespace:task)", task_id); - continue; - }; - - for state in &states { - let mut page = 0u64; - loop { - let req = proto::TaskPageRequest { - namespace: namespace.to_string(), - task: task.to_string(), - page, - page_size: args.page_size, - state: state.to_proto(), - }; - - let resp = match client.get_tasks(Request::new(req)).await { - Ok(r) => r.into_inner(), - Err(e) => { - error!( - "GetTasks failed for {} state {:?}: {}", - task_id, state, e - ); - break; - } - }; - - if resp.tasks.is_empty() { - break; - } - - let key = ID(namespace, task); - let bucket = out_queue.tasks.entry(key).or_default(); - for t in resp.tasks { - bucket.push(StoredTask { - bytes: t.task_payload, - id: t.id, - }); - fetched += 1; - } - - page += 1; - } - } - } - - match postcard::to_allocvec(&out_queue) { - Ok(data) => match File::create(&args.output) { - Ok(mut file) => { - if let Err(e) = file.write_all(&data) { - error!("Failed to write {}: {}", args.output.display(), e); - } else { - info!( - "Export complete. wrote {} task(s) to {}", - fetched, - args.output.display() - ); - } - } - Err(e) => { - error!("Failed to create {}: {}", args.output.display(), e); - } - }, - Err(e) => error!("Failed to serialize export: {}", e), - } - } - Commands::AdminDelete(args) => { - let prepared_headers = build_headers(&api, true); - let interceptor = move |mut req: Request<()>| { - for (key, value) in prepared_headers.iter() { - if let (Ok(key), Ok(value)) = ( - MetadataKey::from_bytes(key.as_bytes()), - MetadataValue::try_from(value.as_str()), - ) { - req.metadata_mut().insert(key, value); - } - } - Ok(req) - }; - - let endpoint = format!("http://{}", api.cfg.config_toml.host); - let channel = match Endpoint::from_shared(endpoint.clone()) { - Ok(ep) => match ep.connect().await { - Ok(ch) => ch, - Err(e) => { - error!("Failed to connect to server {}: {}", endpoint, e); - return; - } - }, - Err(e) => { - error!("Invalid server endpoint {}: {}", endpoint, e); - return; - } - }; - - let mut client = - proto::engine_client::EngineClient::with_interceptor(channel, interceptor); - - let req = proto::TaskSelector { - state: args.state.to_proto(), - namespace: args.namespace.clone(), - task: args.task.clone(), - id: args.id.clone(), - }; - - match client.delete_task(Request::new(req)).await { - Ok(_) => info!( - "Deleted task {} from {}:{} ({:?})", - args.id, args.namespace, args.task, args.state - ), - Err(e) => error!("DeleteTask failed: {}", e), - } - } - Commands::Pack(input) => { - if input.input.exists() { - info!("Packing File: {}", input.input.to_string_lossy()); - match std::fs::read_to_string(&input.input) { - Ok(toml_str) => { - match toml::from_str::(&toml_str) { - Ok(raw) => { - let entries = parse_entries(raw); - for entry in entries { - match api - .task_registry - .get(&ID(entry.namespace.as_str(), entry.id.as_str())) - { - Some(template) => { - match toml::to_string(&entry.data) { - Ok(toml_string) => { - let t = template.from_toml(toml_string); - let key = ID( - entry.namespace.as_str(), - entry.id.as_str(), - ); - let mut vec = api - .task_queue - .tasks - .get(&key) - .cloned() - .unwrap_or_default(); - vec.push(StoredTask { - id: "".into(), //ids are minted on the server - bytes: t.to_bytes(), - }); - api.task_queue.tasks.insert(key, vec); - } - Err(e) => { - error!( - "Failed to convert entry data to TOML string: {}", - e - ); - } - } - } - None => { - error!( - "Template not found for {}:{}", - entry.namespace, entry.id - ); - } - } - } - if input.stream { - match File::create("output.rustforge.bin") { - Ok(mut file) => { - if let Err(e) = write_chunked_header(&mut file) { - error!( - "Failed to write chunked output header: {}", - e - ); - return; - } - - let mut wrote = 0usize; - for ((namespace, task), tasks) in - &api.task_queue.tasks - { - for stored in tasks { - let record = ChunkedTaskRecord { - namespace: namespace.clone(), - task: task.clone(), - id: stored.id.clone(), - payload: stored.bytes.clone(), - }; - if let Err(e) = write_chunked_task_record( - &mut file, &record, - ) { - error!( - "Failed to write chunked output record: {}", - e - ); - return; - } - wrote += 1; - } - } - - if let Err(e) = write_chunked_end(&mut file) { - error!( - "Failed to finalize chunked output: {}", - e - ); - return; - } - - info!( - "Wrote output.rustforge.bin (chunked stream, {} task(s))", - wrote - ); - } - Err(e) => { - error!( - "Failed to create output.rustforge.bin: {}", - e - ); - } - } - } else { - match postcard::to_allocvec(&api.task_queue) { - Ok(data) => { - match File::create("output.rustforge.bin") { - Ok(mut file) => { - if let Err(e) = file.write_all(&data) { - error!( - "Failed to write output.rustforge.bin: {}", - e - ); - } else { - info!("Wrote output.rustforge.bin"); - } - } - Err(e) => { - error!( - "Failed to create output.rustforge.bin: {}", - e - ); - } - } - } - Err(e) => { - error!("Failed to serialize task queue: {}", e); - } - } - } - } - Err(e) => { - error!("Failed to parse input TOML: {}", e); - } - } - } - Err(e) => { - error!("Failed to read input file {}: {}", input.input.display(), e); - } - } - } else { - error!("File does not exist: {}", input.input.to_string_lossy()) - } - } - } - } -} diff --git a/engine/src/bin/server.rs b/engine/src/bin/server.rs index 7b70ff8..b5fd299 100644 --- a/engine/src/bin/server.rs +++ b/engine/src/bin/server.rs @@ -1,770 +1,7 @@ -use engine::{get_auth, get_uid}; -use enginelib::plugin::LibraryMetadata; -use enginelib::{ - Identifier, RawIdentifier, Registry, - api::EngineAPI, - chrono::Utc, - event::{debug, info, warn}, - events::{self, Events, ID}, - plugin::LibraryManager, - task::{SolvedTasks, StoredExecutingTask, StoredTask, Task, TaskQueue}, -}; -use proto::{ - TaskState, - engine_server::{Engine, EngineServer}, -}; -use std::{ - collections::HashMap, - env::consts::OS, - io::Read, - net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4}, - sync::{Arc, RwLock as RS_RwLock}, -}; -use tokio::sync::RwLock; -use tonic::{Request, Response, Status, metadata::MetadataValue, transport::Server}; +use enginelib::api::ServerAPI; -use crate::proto::ModuleInfo; - -mod proto { - tonic::include_proto!("engine"); - pub(crate) const FILE_DESCRIPTOR_SET: &[u8] = - tonic::include_file_descriptor_set!("engine_descriptor"); -} -#[allow(non_snake_case)] -struct EngineService { - pub EngineAPI: Arc>, -} -#[tonic::async_trait] -impl Engine for EngineService { - async fn get_metadata( - &self, - request: tonic::Request, - ) -> Result, Status> { - let api = self.EngineAPI.read().await; - - let modules: Vec = api - .lib_manager - .libraries - .values() - .map(|lib| lib.metadata.clone()) - //Dont show server mods - .filter(|lib| !lib.mod_server) - // vec> --> vec - .map(|f| ModuleInfo { - mod_id: f.mod_id.clone(), - api_version: f.api_version.clone(), - rustc_version: f.rustc_version.clone(), - mod_version: f.mod_version.clone(), - }) - .collect(); - - let res = proto::ServerMetadata { - engine_api: enginelib::GIT_VERSION.to_string(), - mods: modules, - }; - return Ok(Response::new(res)); - } - - async fn check_auth( - &self, - request: tonic::Request, - ) -> Result, Status> { - let challenge = get_auth(&request); - let mut api = self.EngineAPI.write().await; - let db = api.db.clone(); - let output = Events::CheckAdminAuth(&mut api, challenge, ("".into(), "".into()), db); - if !output { - warn!("Auth check failed - permission denied"); - return Err(tonic::Status::permission_denied("Invalid Auth")); - }; - return Ok(tonic::Response::new(proto::Empty {})); - } - async fn delete_task( - &self, - request: tonic::Request, - ) -> Result, Status> { - let mut api = self.EngineAPI.write().await; - let data = request.get_ref(); - let challenge = get_auth(&request); - let db = api.db.clone(); - let id = ID(&data.namespace, &data.task); - - let output = Events::CheckAdminAuth(&mut api, challenge, ("".into(), "".into()), db); - if !output { - warn!("Auth check failed - permission denied"); - return Err(tonic::Status::permission_denied("Invalid Auth")); - }; - // Generic helper for removing a task by id from a collection, using an id extractor closure - fn delete_task_from_collection( - collection: &mut HashMap<(String, String), Vec>, - id: &(String, String), - task_id: &str, - state_name: &str, - namespace: &str, - task: &str, - id_extractor: F, - ) -> Result<(), Status> - where - F: Fn(&T) -> &str, - { - match collection.get_mut(id) { - Some(query) => { - let orig_len = query.len(); - query.retain(|f| id_extractor(f) != task_id); - if query.len() == orig_len { - info!( - "DeleteTask: Task with id {} not found in {} state for namespace: {}, task: {}", - task_id, state_name, namespace, task - ); - return Err(Status::not_found(format!( - "Task with id {} not found in {} state", - task_id, state_name - ))); - } - Ok(()) - } - None => { - info!( - "DeleteTask: No tasks found in {} state for namespace: {}, task: {}", - state_name, namespace, task - ); - Err(Status::not_found(format!( - "No tasks found in {} state for given namespace and task", - state_name - ))) - } - } - } - - // Use the helper for each state - let result = match data.state() { - TaskState::Processing => delete_task_from_collection( - &mut api.executing_tasks.tasks, - &id, - &data.id, - "Processing", - &data.namespace, - &data.task, - |f| &f.id, - ), - TaskState::Solved => delete_task_from_collection( - &mut api.solved_tasks.tasks, - &id, - &data.id, - "Solved", - &data.namespace, - &data.task, - |f| &f.id, - ), - TaskState::Queued => delete_task_from_collection( - &mut api.task_queue.tasks, - &id, - &data.id, - "Queued", - &data.namespace, - &data.task, - |f| &f.id, - ), - }; - - if let Err(e) = result { - return Err(e); - } - - // Sync running memory into DB - EngineAPI::sync_db(&mut api); - info!( - "DeleteTask: Successfully deleted task with id {} in state {:?} for namespace: {}, task: {}", - data.id, - data.state(), - data.namespace, - data.task - ); - Ok(tonic::Response::new(proto::Empty {})) - } - /// Retrieves a paginated list of tasks filtered by namespace, task name, and state. - /// - /// Authenticates the request and, if authorized, returns tasks in the specified state - /// (`Processing`, `Queued`, or `Solved`) for the given namespace and task name. The results - /// are sorted by task ID and paginated according to the requested page and page size. - /// - /// Returns a `TaskPage` containing the filtered tasks and pagination metadata, or a - /// permission denied error if authentication fails. - /// - /// # Examples - /// - /// ``` - /// // Example usage within a tonic gRPC client context: - /// let request = proto::TaskPageRequest { - /// namespace: "example_ns".to_string(), - /// task: "example_task".to_string(), - /// state: proto::TaskState::Queued as i32, - /// page: 0, - /// page_size: 10, - /// }; - /// let response = engine_client.get_tasks(request).await?; - /// assert!(response.get_ref().tasks.len() <= 10); - /// ``` - async fn get_tasks( - &self, - request: tonic::Request, - ) -> std::result::Result, tonic::Status> { - let mut api = self.EngineAPI.write().await; - let challenge = get_auth(&request); - - let db = api.db.clone(); - if !Events::CheckAdminAuth(&mut api, challenge, ("".into(), "".into()), db) { - info!("GetTask denied due to Invalid Auth"); - return Err(Status::permission_denied("Invalid authentication")); - }; - let data = request.get_ref(); - - let q: Vec = match data.clone().state() { - TaskState::Processing => { - match api - .executing_tasks - .tasks - .get(&(data.namespace.clone(), data.task.clone())) - { - Some(tasks) => { - let mut task_refs: Vec<_> = tasks.iter().collect(); - task_refs.sort_by_key(|f| &f.id); - task_refs - .iter() - .map(|f| proto::Task { - id: f.id.clone(), - task_id: format!("{}:{}", data.namespace, data.task), - task_payload: f.bytes.clone(), - payload: Vec::new(), - }) - .collect() - } - None => { - info!( - "Namespace {:?} and task {:?} not found in Processing state", - data.namespace, data.task - ); - Vec::new() - } - } - } - TaskState::Queued => { - match api - .task_queue - .tasks - .get(&(data.namespace.clone(), data.task.clone())) - { - Some(tasks) => { - let mut d = tasks.clone(); - d.sort_by_key(|f| f.id.clone()); - d.iter() - .map(|f| proto::Task { - id: f.id.clone(), - task_id: format!("{}:{}", data.namespace, data.task), - task_payload: f.bytes.clone(), - payload: Vec::new(), - }) - .collect() - } - None => { - info!( - "Namespace {:?} and task {:?} not found in Queued state", - data.namespace, data.task - ); - Vec::new() - } - } - } - TaskState::Solved => { - match api - .solved_tasks - .tasks - .get(&(data.namespace.clone(), data.task.clone())) - { - Some(tasks) => { - let mut d = tasks.clone(); - d.sort_by_key(|f| f.id.clone()); - d.iter() - .map(|f| proto::Task { - id: f.id.clone(), - task_id: format!("{}:{}", data.namespace, data.task), - task_payload: f.bytes.clone(), - payload: Vec::new(), - }) - .collect() - } - None => { - info!( - "Namespace {:?} and task {:?} not found in Solved state", - data.namespace, data.task - ); - Vec::new() - } - } - } - }; - let index = data.page * data.page_size as u64; - let end = index + (api.cfg.config_toml.pagination_limit.min(data.page_size) as u64); - let final_vec: Vec<_> = q - .iter() - .skip(index as usize) - .take(data.page_size as usize) - .cloned() - .collect(); - return Ok(tonic::Response::new(proto::TaskPage { - namespace: data.namespace.clone(), - task: data.task.clone(), - page: data.page, - page_size: data.page_size, - state: data.state, - tasks: final_vec, - })); - } - /// Handles custom gRPC messages with admin-level authentication. - /// - /// Processes a CGRPC request by verifying admin credentials and dispatching the event payload to the appropriate handler. Returns the processed event payload in the response. If authentication fails, returns a permission denied error. - /// - /// # Returns - /// A `Cgrpcmsg` response containing the processed event payload, or a permission denied gRPC status on failed authentication. - /// - /// # Examples - /// - /// ``` - /// // Example usage within a gRPC client context: - /// let request = proto::Cgrpcmsg { - /// handler_mod_id: "mod".to_string(), - /// handler_id: "handler".to_string(), - /// event_payload: vec![1, 2, 3], - /// // ... other fields ... - /// }; - /// let response = engine_service.cgrpc(tonic::Request::new(request)).await?; - /// assert_eq!(response.get_ref().handler_mod_id, "mod"); - /// ``` - async fn cgrpc( - &self, - request: tonic::Request, - ) -> std::result::Result, tonic::Status> { - info!( - "CGRPC request received for handler: {}:{}", - request.get_ref().handler_mod_id, - request.get_ref().handler_id - ); - let mut api = self.EngineAPI.write().await; - let challenge = get_auth(&request); - let db = api.db.clone(); - debug!("Checking admin authentication for CGRPC request"); - let output = Events::CheckAdminAuth( - &mut api, - challenge, - ( - request.get_ref().handler_mod_id.clone(), - request.get_ref().handler_id.clone(), - ), - db, - ); - if !output { - warn!("CGRPC auth check failed - permission denied"); - return Err(tonic::Status::permission_denied("Invalid CGRPC Auth")); - }; - let out = Arc::new(std::sync::RwLock::new(Vec::new())); - debug!("Dispatching CGRPC event to handler"); - Events::CgrpcEvent( - &mut api, - ID("engine_core", "grpc"), - request.get_ref().event_payload.clone(), - out.clone(), - ); - let mut res = request.get_ref().clone(); - res.event_payload = match out.read() { - Ok(g) => g.clone(), - Err(_) => { - warn!("CGRPC response lock poisoned, returning empty payload"); - Vec::new() - } - }; - info!("CGRPC request processed successfully"); - return Ok(tonic::Response::new(res)); - } - async fn aquire_task_reg( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let uid = get_uid(&request); - let challenge = get_auth(&request); - info!("Task registry request received from user: {}", uid); - let mut api = self.EngineAPI.write().await; - let db = api.db.clone(); - - debug!("Validating authentication for task registry request"); - if !Events::CheckAuth(&mut api, uid.clone(), challenge, db) { - info!( - "Task registry request denied - invalid authentication for user: {}", - uid - ); - return Err(Status::permission_denied("Invalid authentication")); - }; - let mut tasks: Vec = Vec::new(); - for (k, v) in &api.task_registry.tasks { - let js: Vec = vec![k.0.clone(), k.1.clone()]; - let jstr = js.join(":"); - tasks.push(jstr); - } - info!("Returning task registry with {} tasks", tasks.len()); - let response = proto::TaskRegistry { tasks }; - Ok(tonic::Response::new(response)) - } - - async fn aquire_task( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let challenge = get_auth(&request); - let task_id = request.get_ref().task_id.clone(); - let uid = get_uid(&request); - info!( - "Task acquisition request received from user: {} for task: {}", - uid, task_id - ); - - { - let mut api = self.EngineAPI.write().await; - let db = api.db.clone(); - debug!("Validating authentication for task acquisition"); - if !Events::CheckAuth(&mut api, uid.clone(), challenge, db) { - info!( - "Task acquisition denied - invalid authentication for user: {}", - uid - ); - return Err(Status::permission_denied("Invalid authentication")); - }; - } - - let (namespace, task_name) = task_id.split_once(':').ok_or_else(|| { - info!("Invalid task ID format: {}", task_id); - Status::invalid_argument("Invalid task ID format, expected 'namespace:task'") - })?; - - debug!("Looking up task definition for {}:{}", namespace, task_name); - let key = ID(namespace, task_name); - - { - let api = self.EngineAPI.read().await; - if api.task_registry.get(&key).is_none() { - warn!( - "Task acquisition failed - task does not exist: {}:{}", - namespace, task_name - ); - return Err(Status::invalid_argument("Task Does not Exist")); - } - if Events::ServerBeforeTaskAcquire(&api, uid.clone(), task_id.clone()) { - info!( - "ServerBeforeTaskAcquire cancelled for user: {} task: {}", - uid, task_id - ); - return Err(Status::aborted( - "Task acquire cancelled by server event handler", - )); - } - } - - let (ttask, tasks_key_state, exec_key_state, db) = { - let mut api = self.EngineAPI.write().await; - - let queue = api - .task_queue - .tasks - .get_mut(&key) - .ok_or_else(|| Status::not_found("No queued tasks available"))?; - - if queue.is_empty() { - info!("No queued tasks for {}:{}", namespace, task_name); - return Err(Status::not_found("No queued tasks available")); - } - - let ttask = queue.remove(0); - let task_payload = ttask.bytes.clone(); - - api.executing_tasks - .tasks - .entry(key.clone()) - .or_default() - .push(enginelib::task::StoredExecutingTask { - bytes: task_payload, - user_id: uid.clone(), - given_at: Utc::now(), - id: ttask.id.clone(), - }); - - let tasks_key_state = api.task_queue.tasks.get(&key).cloned().unwrap_or_default(); - let exec_key_state = api - .executing_tasks - .tasks - .get(&key) - .cloned() - .unwrap_or_default(); - let db = api.db.clone(); - (ttask, tasks_key_state, exec_key_state, db) - }; - - let tasks_op = EngineAPI::state_op_tasks(&key, &tasks_key_state) - .map_err(|e| Status::internal(format!("Serialization error: {}", e)))?; - let exec_op = EngineAPI::state_op_executing(&key, &exec_key_state) - .map_err(|e| Status::internal(format!("Serialization error: {}", e)))?; - - EngineAPI::apply_batch_ops(&db, vec![tasks_op, exec_op]) - .map_err(|e| Status::internal(format!("DB insert error: {}", e)))?; - - { - let api = self.EngineAPI.read().await; - Events::ServerTaskAcquired(&api, uid.clone(), task_id.clone(), ttask.id.clone()); - } - - Ok(tonic::Response::new(proto::Task { - id: ttask.id, - task_id, - task_payload: ttask.bytes, - payload: Vec::new(), - })) - } - async fn publish_task( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let challenge = get_auth(&request); - let uid = get_uid(&request); - - let task_id = request.get_ref().task_id.clone(); - let instance_id = request.get_ref().id.clone(); - let payload_for_event = Arc::new(std::sync::RwLock::new( - request.get_ref().task_payload.clone(), - )); - - { - let mut api = self.EngineAPI.write().await; - let db = api.db.clone(); - if !Events::CheckAuth(&mut api, uid.clone(), challenge, db) { - info!("Aquire Task denied due to Invalid Auth"); - return Err(Status::permission_denied("Invalid authentication")); - }; - } - - { - let api = self.EngineAPI.read().await; - if Events::ServerBeforeTaskPublish( - &api, - uid.clone(), - task_id.clone(), - instance_id.clone(), - payload_for_event.clone(), - ) { - info!( - "ServerBeforeTaskPublish cancelled for user: {} task: {}", - uid, task_id - ); - return Err(Status::aborted( - "Task publish cancelled by server event handler", - )); - } - } - - let publish_payload = payload_for_event - .read() - .map(|p| p.clone()) - .map_err(|_| Status::internal("Task publish payload lock poisoned"))?; - - let (namespace, task_name) = task_id - .split_once(':') - .ok_or_else(|| Status::invalid_argument("Invalid Params"))?; - let key = ID(namespace, task_name); - - let reg_tsk = { - let api = self.EngineAPI.read().await; - if !api.task_registry.tasks.contains_key(&key) { - warn!( - "Task acquisition failed - task does not exist: {}:{}", - namespace, task_name - ); - return Err(Status::invalid_argument("Task Does not Exist")); - } - - match api.task_registry.get(&key) { - Some(r) => r, - None => { - warn!("Task registry missing for {}:{}", namespace, task_name); - return Err(Status::invalid_argument("Task Does not Exist")); - } - } - }; - - if !reg_tsk.verify(publish_payload.clone()) { - info!("Failed to parse task"); - return Err(Status::invalid_argument("Failed to parse given task bytes")); - } - - let (solved_id, exec_key_state, solved_key_state, db) = { - let mut api = self.EngineAPI.write().await; - - let solved_id = { - let exec_tasks = api - .executing_tasks - .tasks - .get_mut(&key) - .ok_or_else(|| tonic::Status::not_found("Invalid taskid or userid"))?; - - let idx = exec_tasks - .iter() - .position(|f| f.id == instance_id && f.user_id == uid) - .ok_or_else(|| tonic::Status::not_found("Invalid taskid or userid"))?; - - exec_tasks.remove(idx).id - }; - - api.solved_tasks.tasks.entry(key.clone()).or_default().push( - enginelib::task::StoredTask { - bytes: publish_payload.clone(), - id: solved_id.clone(), - }, - ); - - let exec_key_state = api - .executing_tasks - .tasks - .get(&key) - .cloned() - .unwrap_or_default(); - let solved_key_state = api - .solved_tasks - .tasks - .get(&key) - .cloned() - .unwrap_or_default(); - let db = api.db.clone(); - - (solved_id, exec_key_state, solved_key_state, db) - }; - - let exec_op = EngineAPI::state_op_executing(&key, &exec_key_state) - .map_err(|e| Status::internal(format!("Serialization error: {}", e)))?; - let solved_op = EngineAPI::state_op_solved(&key, &solved_key_state) - .map_err(|e| Status::internal(format!("Serialization error: {}", e)))?; - - EngineAPI::apply_batch_ops(&db, vec![exec_op, solved_op]) - .map_err(|e| Status::internal(format!("DB insert error: {}", e)))?; - - { - let api = self.EngineAPI.read().await; - Events::ServerTaskPublished(&api, uid.clone(), task_id.clone(), solved_id); - } - - info!("Task published successfully: {} by user: {}", task_id, uid); - Ok(tonic::Response::new(proto::Empty {})) - } - async fn create_task( - &self, - request: tonic::Request, - ) -> Result, tonic::Status> { - let mut api = self.EngineAPI.write().await; - let challenge = get_auth(&request); - let uid = get_uid(&request); - let db = api.db.clone(); - if !Events::CheckAuth(&mut api, uid, challenge, db) { - //TODO: change to AdminSpecific Auth - info!("Create Task denied due to Invalid Auth"); - return Err(Status::permission_denied("Invalid authentication")); - }; - let task = request.get_ref(); - let task_id = task.task_id.clone(); - let payload_for_event = Arc::new(std::sync::RwLock::new(task.task_payload.clone())); - if Events::ServerBeforeTaskCreate(&api, task_id.clone(), payload_for_event.clone()) { - info!("ServerBeforeTaskCreate cancelled for task: {}", task_id); - return Err(Status::aborted( - "Task create cancelled by server event handler", - )); - } - let task_payload = payload_for_event - .read() - .map(|p| p.clone()) - .map_err(|_| Status::internal("Task create payload lock poisoned"))?; - - let parts: Vec<&str> = task_id.splitn(2, ':').collect(); - if parts.len() != 2 || parts[0].is_empty() || parts[1].is_empty() { - return Err(Status::invalid_argument( - "Invalid task ID format, expected 'namespace:task'", - )); - } - let id: Identifier = (parts[0].to_string(), parts[1].to_string()); - let tsk_reg = api.task_registry.get(&id); - if let Some(tsk_reg) = tsk_reg { - if !tsk_reg.clone().verify(task_payload.clone()) { - warn!("Failed to parse given task bytes"); - return Err(Status::invalid_argument("Failed to parse given task bytes")); - } - let tbp_tsk = StoredTask { - bytes: task_payload.clone(), - id: druid::Druid::default().to_hex(), - }; - api.task_queue - .tasks - .entry(id.clone()) - .or_default() - .push(tbp_tsk.clone()); - - let tasks_key_state = api.task_queue.tasks.get(&id).cloned().unwrap_or_default(); - let task_op = EngineAPI::state_op_tasks(&id, &tasks_key_state) - .map_err(|e| Status::internal(format!("Serialization error: {}", e)))?; - if let Err(e) = EngineAPI::apply_batch_ops(&api.db, vec![task_op]) { - return Err(Status::internal(format!("DB insert error: {}", e))); - } - Events::ServerTaskCreated( - &api, - task_id.clone(), - tbp_tsk.id.clone(), - Arc::new(std::sync::RwLock::new(tbp_tsk.bytes.clone())), - ); - return Ok(tonic::Response::new(proto::Task { - id: tbp_tsk.id.clone(), - task_id: task_id.clone(), - payload: Vec::new(), - task_payload: tbp_tsk.bytes.clone(), - })); - } - Err(tonic::Status::aborted("Error")) - } -} - -#[tokio::main(flavor = "multi_thread", worker_threads = 8)] -async fn main() -> Result<(), Box> { - let mut api = EngineAPI::default(); - EngineAPI::init(&mut api); - Events::init_auth(&mut api); - Events::StartEvent(&mut api); - Events::ServerStart(&api); - let addr = api - .cfg - .config_toml - .host - .parse() - .unwrap_or(SocketAddr::V4(SocketAddrV4::new( - Ipv4Addr::new(127, 0, 0, 1), - 50051, - ))); - let apii = Arc::new(RwLock::new(api)); - EngineAPI::init_chron(apii.clone()); - let engine = EngineService { EngineAPI: apii }; - - // Build reflection service, mapping its concrete error into Box - let reflection_service = tonic_reflection::server::Builder::configure() - .register_encoded_file_descriptor_set(proto::FILE_DESCRIPTOR_SET) - .build_v1() - .map_err(|e| Box::new(e) as Box)?; - - // Start server and map transport errors into Box so `?` works with our return type. - Server::builder() - .add_service(reflection_service) - .add_service(EngineServer::new(engine)) - .serve(addr) - .await - .map_err(|e| Box::new(e) as Box)?; - - Ok(()) +fn main() { + let mut api = ServerAPI::default(); + // let server = enginelib::server::RPC::new(&mut api); + // server.run(); } diff --git a/engine/src/lib.rs b/engine/src/lib.rs deleted file mode 100644 index 5b529f0..0000000 --- a/engine/src/lib.rs +++ /dev/null @@ -1,17 +0,0 @@ -use tonic::Request; - -pub fn get_uid(req: &Request) -> String { - req.metadata() - .get("uid") - .and_then(|v| v.to_str().ok()) - .map(|s| s.to_string()) - .unwrap_or_default() -} - -pub fn get_auth(req: &Request) -> String { - req.metadata() - .get("authorization") - .and_then(|v| v.to_str().ok()) - .map(|s| s.to_string()) - .unwrap_or_default() -} diff --git a/enginelib/Cargo.toml b/enginelib/Cargo.toml index 0481fa3..e2512fe 100644 --- a/enginelib/Cargo.toml +++ b/enginelib/Cargo.toml @@ -16,16 +16,19 @@ toml = { workspace = true } tracing = "0.1.44" tracing-subscriber = "0.3.22" chrono = { version = "0.4.44", features = ["serde"] } -sled = "0.34.7" tokio = { version = "1.50.0", features = ["full"] } postcard = { version = "1.1.3", features = ["use-std"] } inventory = "0.3.22" crossbeam = "0.8.4" +dashmap = "6.2.1" +async-channel = "2.5.0" +druid = {git="https://github.com/voltaero/druid.git"} [build-dependencies] vergen-gix = { version = "9.1.0", features = ["build", "cargo", "rustc"] } -[profile.release] -codegen-units = 1 # Make builds deterministic -[profile.dev] -codegen-units = 1 # Make builds deterministic +# Profiles moved to the workspace root Cargo.toml — Cargo ignores profile +# tables in non-root member manifests. [dev-dependencies] tracing-test = "0.2.6" +[dependencies.rust-rocksdb] +features = ["jemalloc"] +version = "0.51.0" diff --git a/enginelib/src/api.rs b/enginelib/src/api.rs index aab43f8..f9de1b7 100644 --- a/enginelib/src/api.rs +++ b/enginelib/src/api.rs @@ -1,280 +1,45 @@ use chrono::Utc; -use crossbeam::queue::ArrayQueue; -use tokio::{spawn, sync::RwLock, time::interval}; -use tracing::{Level, debug, error, info, instrument}; +use dashmap::{DashMap, DashSet}; +use tokio::{spawn, time::interval}; +use tracing::{Level, debug, info, instrument}; +use crate::task::Task; use crate::{ Identifier, Registry, config::Config, event::{EngineEventHandlerRegistry, EventBus}, plugin::LibraryManager, - task::{LeasedTaskQueue, StoredTask, Task, TaskQueue}, }; pub use postcard; pub use postcard::from_bytes; pub use postcard::to_allocvec; use std::{ - collections::{HashMap, HashSet}, - sync::Arc, - time::Duration, + collections::HashMap, + sync::RwLock, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + time::{Duration, SystemTime, UNIX_EPOCH}, }; pub struct ServerAPI { - pub cfg: Config, // RW - pub task_queue: TaskQueue, // RW - pub leased_tasks: LeasedTaskQueue, // RW + pub cfg: RwLock, // RW + pub event_bus: EventBus, // RW + pub lib_manager: LibraryManager, // RW + pub db: rust_rocksdb::DB, + // pub task_queue: TaskQueue, // RW + // pub leased_tasks: LeasedTaskQueue, // RW pub task_registry: EngineTaskRegistry, // RW - pub event_bus: EventBus, // RW - pub db: sled::Db, // R - pub lib_manager: LibraryManager, // RW - pub client: bool, // RW } - impl Default for ServerAPI { fn default() -> Self { - Self { - cfg: Config::default(), - task_queue: TaskQueue::default(), - db: sled::open("engine_db").unwrap(), - lib_manager: LibraryManager::default(), - task_registry: EngineTaskRegistry::default(), - event_bus: EventBus { - event_handler_registry: EngineEventHandlerRegistry { - event_handlers: HashMap::new(), - }, - }, - leased_tasks: LeasedTaskQueue::default(), - client: false, - } - } -} -impl ServerAPI { - // pub fn default_client() -> Self { - // Self { - // cfg: Config::default(), - // task_queue: TaskQueue::default(), - // db: sled::open("engine_client_db").unwrap(), - // lib_manager: LibraryManager::default(), - // task_registry: EngineTaskRegistry::default(), - // event_bus: EventBus { - // event_handler_registry: EngineEventHandlerRegistry { - // event_handlers: HashMap::new(), - // }, - // }, - // solved_tasks: SolvedTasks::default(), - // executing_tasks: ExecutingTaskQueue::default(), - // client: true, - // } - //} - pub fn test_default() -> Self { - // `sled::Config::temporary(true)` defaults to `/dev/shm` on Linux when no path is set. - // Some environments deny writes there, so force a unique temp path. - use std::sync::atomic::{AtomicUsize, Ordering}; - - static DB_COUNTER: AtomicUsize = AtomicUsize::new(0); - let db_id = DB_COUNTER.fetch_add(1, Ordering::Relaxed); - let db_path = std::env::temp_dir().join(format!( - "enginelib-test-db-{}-{}", - std::process::id(), - db_id - )); - - Self { - client: false, - cfg: Config::new(), - leased_tasks: LeasedTaskQueue::default(), - task_queue: TaskQueue::default(), - db: sled::Config::new() - .path(db_path) - .temporary(true) - .flush_every_ms(None) - .open() - .unwrap(), - lib_manager: LibraryManager::default(), - task_registry: EngineTaskRegistry::default(), - event_bus: EventBus { - event_handler_registry: EngineEventHandlerRegistry { - event_handlers: HashMap::new(), - }, - }, - } - } - pub fn init(api: &mut Self) { - Self::setup_logger(); - api.cfg = Config::new(); - Self::init_db(api); - let mut new_lib_manager = LibraryManager::default(); - new_lib_manager.load_modules(api); - api.lib_manager = new_lib_manager; - for (id, _tsk) in api.task_registry.tasks.iter() { - api.task_queue - .tasks - .entry(id.clone()) - .insert_entry(ArrayQueue::new( - api.cfg.config_toml.task_block_size as usize, - )); - api.leased_tasks.tasks.entry(id.clone()).or_default(); - } - - Self::init_events(api); - } - - fn init_events(api: &mut Self) { - crate::event::register_inventory_handlers(api); - } - pub fn init_chron(api: Arc>) { - let t = api.try_read().unwrap().cfg.config_toml.clean_tasks; - spawn(clear_sled_periodically(api, t)); - } - const TASKS_PREFIX: &'static str = "tasks:"; - const LEASING_PREFIX: &'static str = "leasing:"; - const SOLVED_PREFIX: &'static str = "solved:"; - - fn state_key(prefix: &str, task_id: &Identifier, id: String) -> Vec { - format!("{}{}\u{1f}{}:{}", prefix, task_id.0, task_id.1, id).into_bytes() - } - - fn parse_state_key(prefix: &str, key: &[u8]) -> Option { - let key = std::str::from_utf8(key).ok()?; - let rest = key.strip_prefix(prefix)?; - let (task_id, id) = rest.split_once(":")?; - let (namespace, task) = task_id.split_once('\u{1f}')?; - Some((namespace.to_string(), task.to_string())) - } - - // pub fn apply_batch_entries( - // db: &sled::Db, - // entries: Vec<(&'static str, Vec)>, - // ) -> sled::Result<()> { - // let mut batch = sled::Batch::default(); - // for (key, value) in entries { - // batch.insert(key, value); - // } - // db.apply_batch(batch) - // } - - // pub fn apply_batch_ops( - // db: &sled::Db, - // ops: Vec<(Vec, Option>)>, - // ) -> sled::Result<()> { - // let mut batch = sled::Batch::default(); - // for (key, value) in ops { - // match value { - // Some(v) => batch.insert(key, v), - // None => batch.remove(key), - // } - // } - // db.apply_batch(batch) - // } - - // pub fn state_op_tasks( - // id: &Identifier, - // value: &Vec, - // ) -> Result<(Vec, Option>), postcard::Error> { - // if value.is_empty() { - // Ok((Self::state_key(Self::TASKS_PREFIX, id), None)) - // } else { - // Ok(( - // Self::state_key(Self::TASKS_PREFIX, id), - // Some(postcard::to_allocvec(value)?), - // )) - // } - // } - - // pub fn state_op_executing( - // id: &Identifier, - // value: &Vec, - // ) -> Result<(Vec, Option>), postcard::Error> { - // if value.is_empty() { - // Ok((Self::state_key(Self::LEASING_PREFIX, id), None)) - // } else { - // Ok(( - // Self::state_key(Self::LEASING_PREFIX, id), - // Some(postcard::to_allocvec(value)?), - // )) - // } - // } - - // pub fn state_op_solved( - // id: &Identifier, - // value: &Vec, - // ) -> Result<(Vec, Option>), postcard::Error> { - // if value.is_empty() { - // Ok((Self::state_key(Self::SOLVED_PREFIX, id), None)) - // } else { - // Ok(( - // Self::state_key(Self::SOLVED_PREFIX, id), - // Some(postcard::to_allocvec(value)?), - // )) - // } - // } - - // pub fn sync_db(api: &mut ServerAPI) { - // // IF THIS FN CAUSES PANIC SOMETHING IS VERY BROKEN - // let mut ops: Vec<(Vec, Option>)> = Vec::new(); + let path = "./engine_db"; + let mut opts = rust_rocksdb::Options::default(); + opts.create_if_missing(true); - // for prefix in [ - // Self::TASKS_PREFIX, - // Self::LEASING_PREFIX, - // Self::SOLVED_PREFIX, - // ] { - // for item in api.db.scan_prefix(prefix.as_bytes()) { - // if let Ok((key, _)) = item { - // ops.push((key.to_vec(), None)); - // } - // } - // } + let db = rust_rocksdb::DB::open(&opts, path).unwrap(); - // for (id, tasks) in &api.task_queue.tasks { - // ops.push(Self::state_op_tasks(id, tasks).unwrap()); - // } - // for (id, tasks) in &api.executing_tasks.tasks { - // ops.push(Self::state_op_executing(id, tasks).unwrap()); - // } - // for (id, tasks) in &api.solved_tasks.tasks { - // ops.push(Self::state_op_solved(id, tasks).unwrap()); - // } - - // Self::apply_batch_ops(&api.db, ops).unwrap(); - // debug!("Synced in-memory state to keyed sled storage"); - // } - - fn init_db(api: &mut ServerAPI) { - api.task_queue = TaskQueue::default(); - api.leased_tasks = LeasedTaskQueue::default(); - - for item in api.db.scan_prefix(Self::TASKS_PREFIX.as_bytes()) { - if let Ok((key, value)) = item { - if let Some(id) = Self::parse_state_key(Self::TASKS_PREFIX, &key) { - if let Ok(tasks) = postcard::from_bytes::(&value) { - api.task_queue.tasks.get(&id).unwrap().push(Arc::new(tasks)); - } - } - } - } - - for item in api.db.scan_prefix(Self::LEASING_PREFIX.as_bytes()) { - if let Ok((key, value)) = item { - if let Some(id) = Self::parse_state_key(Self::LEASING_PREFIX, &key) { - if let Ok(tasks) = postcard::from_bytes::>(&value) { - api.executing_tasks.tasks.insert(id, tasks); - } - } - } - } - - for item in api.db.scan_prefix(Self::SOLVED_PREFIX.as_bytes()) { - if let Ok((key, value)) = item { - if let Some(id) = Self::parse_state_key(Self::SOLVED_PREFIX, &key) { - if let Ok(tasks) = postcard::from_bytes::>(&value) { - api.solved_tasks.tasks.insert(id, tasks); - } - } - } - } - } - - pub fn setup_logger() { use std::sync::OnceLock; static INIT: OnceLock<()> = OnceLock::new(); @@ -294,8 +59,20 @@ impl ServerAPI { // builds the subscriber. .try_init(); }); + + let mut k = Self { + cfg: RwLock::new(Config::default()), + event_bus: EventBus::default(), + lib_manager: LibraryManager::default(), + db: db, + task_registry: EngineTaskRegistry::default(), + }; + LibraryManager::load_modules(&mut k); + crate::event::register_inventory_handlers(&mut k); + k } } + #[derive(Default, Clone, Debug)] pub struct EngineTaskRegistry { pub tasks: HashMap>, @@ -315,89 +92,3 @@ impl Registry for EngineTaskRegistry { self.tasks.get(identifier).map(|obj| obj.clone_box()) } } - -pub async fn clear_sled_periodically(api: Arc>, n_minutes: u64) { - info!("Sled Cron Job Started"); - let mut interval = interval(Duration::from_secs(n_minutes * 60)); - loop { - interval.tick().await; - info!("Purging Unsolved Tasks"); - - let now = Utc::now().timestamp(); - let mut rw_api = api.write().await; - - let mut moved_tasks: Vec<(Identifier, StoredTask)> = Vec::new(); - let mut touched_exec: HashSet = HashSet::new(); - - for (id, task_list) in rw_api.executing_tasks.tasks.iter_mut() { - let before_len = task_list.len(); - task_list.retain(|info| { - let age = now - info.given_at.timestamp(); - if age > 3600 { - info!("Task {:?} is older than an hour! Moving...", info); - moved_tasks.push(( - id.clone(), - StoredTask { - id: info.id.clone(), - bytes: info.bytes.clone(), - }, - )); - false - } else { - true - } - }); - - if task_list.len() != before_len { - touched_exec.insert(id.clone()); - } - } - - let mut touched_tasks: HashSet = HashSet::new(); - for (id, task) in moved_tasks { - rw_api - .task_queue - .tasks - .entry(id.clone()) - .or_default() - .push(task); - touched_tasks.insert(id); - } - - if touched_exec.is_empty() && touched_tasks.is_empty() { - continue; - } - - let mut ops: Vec<(Vec, Option>)> = Vec::new(); - - for id in touched_exec { - let value = rw_api - .executing_tasks - .tasks - .get(&id) - .cloned() - .unwrap_or_default(); - match ServerAPI::state_op_executing(&id, &value) { - Ok(op) => ops.push(op), - Err(e) => error!("Failed to serialize executing_tasks entry: {:?}", e), - } - } - - for id in touched_tasks { - let value = rw_api - .task_queue - .tasks - .get(&id) - .cloned() - .unwrap_or_default(); - match ServerAPI::state_op_tasks(&id, &value) { - Ok(op) => ops.push(op), - Err(e) => error!("Failed to serialize tasks entry: {:?}", e), - } - } - - if let Err(e) = ServerAPI::apply_batch_ops(&rw_api.db, ops) { - error!("Failed to update task state in Sled batch: {:?}", e); - } - } -} diff --git a/enginelib/src/config.rs b/enginelib/src/config.rs index 2d46cb5..7d94405 100644 --- a/enginelib/src/config.rs +++ b/enginelib/src/config.rs @@ -7,37 +7,15 @@ fn default_host() -> String { "[::1]:50051".into() } -fn default_clean_tasks() -> u64 { - 60 -} -fn default_task_block_size() -> u32 { - 256 -} -fn default_pagination_limit() -> u32 { - u32::MAX -} - #[derive(Debug, Serialize, Deserialize, Clone)] pub struct ConfigTomlServer { - #[serde(default)] - pub cgrpc_token: Option, // Administrator Token, used to invoke cgrpc reqs. If not preset will default to no protection. #[serde(default = "default_host")] pub host: String, - #[serde(default = "default_clean_tasks")] - pub clean_tasks: u64, - #[serde(default = "default_pagination_limit")] - pub pagination_limit: u32, - #[serde(default = "default_task_block_size")] - pub task_block_size: u32, } impl Default for ConfigTomlServer { fn default() -> Self { Self { - cgrpc_token: None, host: default_host(), - clean_tasks: default_clean_tasks(), - pagination_limit: default_pagination_limit(), - task_block_size: default_task_block_size(), } } } diff --git a/enginelib/src/error.rs b/enginelib/src/error.rs new file mode 100644 index 0000000..6699220 --- /dev/null +++ b/enginelib/src/error.rs @@ -0,0 +1,58 @@ +use std::*; +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ErrorKind { + NotFound, + NotSupported, + InvalidArgument, + IOError, + Unknown, +} +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Error { + message: String, +} + +impl Error { + pub fn new(message: String) -> Error { + Error { message } + } + + pub fn into_string(self) -> String { + self.into() + } + + /// Parse corresponding [`ErrorKind`] from error message. + pub fn kind(&self) -> ErrorKind { + match self.message.split(':').next().unwrap_or("") { + "NotFound" => ErrorKind::NotFound, + "Not implemented" => ErrorKind::NotSupported, + "Invalid argument" => ErrorKind::InvalidArgument, + "IO error" => ErrorKind::IOError, + _ => ErrorKind::Unknown, + } + } +} + +impl AsRef for Error { + fn as_ref(&self) -> &str { + &self.message + } +} + +impl From for String { + fn from(e: Error) -> String { + e.message + } +} + +impl std::error::Error for Error { + fn description(&self) -> &str { + &self.message + } +} + +impl fmt::Display for Error { + fn fmt(&self, formatter: &mut fmt::Formatter) -> Result<(), fmt::Error> { + self.message.fmt(formatter) + } +} diff --git a/enginelib/src/event.rs b/enginelib/src/event.rs index b8de754..98e0af6 100644 --- a/enginelib/src/event.rs +++ b/enginelib/src/event.rs @@ -53,7 +53,7 @@ pub trait EventCTX: EventHandler { fn handleCTX(&self, event: &mut C); } - +#[derive(Default, Clone)] pub struct EventBus { pub event_handler_registry: EngineEventHandlerRegistry, } @@ -100,6 +100,14 @@ impl EngineEventHandlerRegistry { } impl EventBus { + pub fn has_handlers(&self, identifier: &Identifier) -> bool { + self.event_handler_registry + .event_handlers + .get(identifier) + .map(|handlers| !handlers.is_empty()) + .unwrap_or(false) + } + pub fn register_handler( &mut self, handler: H, diff --git a/enginelib/src/events/admin_auth_event.rs b/enginelib/src/events/admin_auth_event.rs index c13533d..c4bb152 100644 --- a/enginelib/src/events/admin_auth_event.rs +++ b/enginelib/src/events/admin_auth_event.rs @@ -1,56 +1,56 @@ -use std::sync::{Arc, RwLock}; +// use std::sync::{Arc, RwLock}; -use macros::{Event, event_handler}; -use sled::Db; +// use macros::{Event, event_handler}; +// use sled::Db; -use crate::{Identifier, api::ServerAPI}; +// use crate::{Identifier, api::ServerAPI}; -#[derive(Clone, Debug, Event)] -#[event(namespace = "core", name = "admin_auth_event", cancellable)] -pub struct AdminAuthEvent { - pub cancelled: bool, - pub id: Identifier, - pub payload: String, - pub target: Identifier, - pub db: Db, - pub output: Arc>, -} -// Event trait auto-implemented by derive macro +// #[derive(Clone, Debug, Event)] +// #[event(namespace = "core", name = "admin_auth_event", cancellable)] +// pub struct AdminAuthEvent { +// pub cancelled: bool, +// pub id: Identifier, +// pub payload: String, +// pub target: Identifier, +// pub db: Db, +// pub output: Arc>, +// } +// // Event trait auto-implemented by derive macro -impl AdminAuthEvent { - pub fn fire( - api: &mut ServerAPI, - payload: String, - target: Identifier, - db: Db, - output: Arc>, - ) { - api.event_bus.fire(&mut AdminAuthEvent { - cancelled: false, - id: ("core".to_string(), "admin_auth_event".to_string()), - payload, - target, - db, - output, - }); - } +// impl AdminAuthEvent { +// pub fn fire( +// api: &ServerAPI, +// payload: String, +// target: Identifier, +// db: Db, +// output: Arc>, +// ) { +// api.event_bus.fire(&mut AdminAuthEvent { +// cancelled: false, +// id: ("core".to_string(), "admin_auth_event".to_string()), +// payload, +// target, +// db, +// output, +// }); +// } - pub fn check(api: &mut ServerAPI, payload: String, target: Identifier, db: Db) -> bool { - let output = Arc::new(RwLock::new(false)); - Self::fire(api, payload, target, db, output.clone()); - *output.read().unwrap() - } -} +// pub fn check(api: &ServerAPI, payload: String, target: Identifier, db: Db) -> bool { +// let output = Arc::new(RwLock::new(false)); +// Self::fire(api, payload, target, db, output.clone()); +// *output.read().unwrap() +// } +// } -#[event_handler( - namespace = "core", - name = "admin_auth_event", - ctx = api.cfg.config_toml.cgrpc_token.clone() -)] -fn admin_auth_handler(event: &mut AdminAuthEvent, token: &Option) { - match token.as_deref() { - None => *event.output.write().unwrap() = true, - Some(token) if token == event.payload.as_str() => *event.output.write().unwrap() = true, - Some(_) => {} - } -} +// #[event_handler( +// namespace = "core", +// name = "admin_auth_event", +// ctx = api.cfg.config_toml.cgrpc_token.clone() +// )] +// fn admin_auth_handler(event: &mut AdminAuthEvent, token: &Option) { +// match token.as_deref() { +// None => *event.output.write().unwrap() = true, +// Some(token) if token == event.payload.as_str() => *event.output.write().unwrap() = true, +// Some(_) => {} +// } +// } diff --git a/enginelib/src/events/auth_event.rs b/enginelib/src/events/auth_event.rs index 1056a99..cbb0b7e 100644 --- a/enginelib/src/events/auth_event.rs +++ b/enginelib/src/events/auth_event.rs @@ -2,7 +2,6 @@ use std::sync::{Arc, RwLock}; use crate::{Identifier, api::ServerAPI}; use macros::{Event, event_handler}; -use sled::Db; #[derive(Clone, Debug, Event)] #[event(namespace = "core", name = "auth_event", cancellable)] @@ -11,15 +10,15 @@ pub struct AuthEvent { pub id: Identifier, pub uid: String, pub challenge: String, - pub db: Db, + pub db: Arc, pub output: Arc>, } impl AuthEvent { pub fn fire( - api: &mut ServerAPI, + api: &ServerAPI, uid: String, challenge: String, - db: Db, + db: rust_rocksdb::DB, output: Arc>, ) { api.event_bus.fire(&mut AuthEvent { @@ -27,12 +26,12 @@ impl AuthEvent { id: ("core".to_string(), "auth_event".to_string()), uid, challenge, - db, + db: Arc::new(db), output, }); } - pub fn check(api: &mut ServerAPI, uid: String, challenge: String, db: Db) -> bool { + pub fn check(api: &ServerAPI, uid: String, challenge: String, db: rust_rocksdb::DB) -> bool { let output = Arc::new(RwLock::new(false)); Self::fire(api, uid, challenge, db, output.clone()); *output.read().unwrap() diff --git a/enginelib/src/events/before_task_acquire_event.rs b/enginelib/src/events/before_task_acquire_event.rs deleted file mode 100644 index 9b85683..0000000 --- a/enginelib/src/events/before_task_acquire_event.rs +++ /dev/null @@ -1,27 +0,0 @@ -use macros::Event; - -use crate::{Identifier, api::ServerAPI}; - -#[derive(Clone, Debug, Event)] -#[event(namespace = "client", name = "before_task_acquire", cancellable)] -pub struct BeforeTaskAcquireEvent { - pub cancelled: bool, - pub id: Identifier, - pub task_id: String, -} - -impl BeforeTaskAcquireEvent { - pub fn fire(api: &ServerAPI, task_id: String) -> Self { - let mut event = BeforeTaskAcquireEvent { - cancelled: false, - id: ("client".to_string(), "before_task_acquire".to_string()), - task_id, - }; - api.event_bus.fire(&mut event); - event - } - - pub fn check(api: &ServerAPI, task_id: String) -> bool { - Self::fire(api, task_id).cancelled - } -} diff --git a/enginelib/src/events/before_task_block_acquire_event.rs b/enginelib/src/events/before_task_block_acquire_event.rs new file mode 100644 index 0000000..13a79f1 --- /dev/null +++ b/enginelib/src/events/before_task_block_acquire_event.rs @@ -0,0 +1,30 @@ +use macros::Event; + +use crate::{Identifier, api::ServerAPI}; + +#[derive(Clone, Debug, Event)] +#[event(namespace = "client", name = "before_task_block_acquire", cancellable)] +pub struct BeforeTaskBlockAcquireEvent { + pub cancelled: bool, + pub id: Identifier, + pub task_ids: Vec, +} + +impl BeforeTaskBlockAcquireEvent { + pub fn fire(api: &ServerAPI, task_ids: Vec) -> Self { + let mut event = BeforeTaskBlockAcquireEvent { + cancelled: false, + id: ( + "client".to_string(), + "before_task_block_acquire".to_string(), + ), + task_ids, + }; + api.event_bus.fire(&mut event); + event + } + + pub fn check(api: &ServerAPI, task_ids: Vec) -> bool { + Self::fire(api, task_ids).cancelled + } +} diff --git a/enginelib/src/events/before_task_block_execute_event.rs b/enginelib/src/events/before_task_block_execute_event.rs new file mode 100644 index 0000000..5da0ccc --- /dev/null +++ b/enginelib/src/events/before_task_block_execute_event.rs @@ -0,0 +1,46 @@ +use std::sync::{Arc, RwLock}; + +use macros::Event; + +use crate::{Identifier, api::ServerAPI}; + +#[derive(Clone, Debug, Event)] +#[event(namespace = "client", name = "before_task_block_execute", cancellable)] +pub struct BeforeTaskBlockExecuteEvent { + pub cancelled: bool, + pub id: Identifier, + pub task_id: String, + pub instance_ids: Vec, + pub payloads: Vec>>>, +} + +impl BeforeTaskBlockExecuteEvent { + pub fn fire( + api: &ServerAPI, + task_id: String, + instance_ids: Vec, + payloads: Vec>>>, + ) -> Self { + let mut event = BeforeTaskBlockExecuteEvent { + cancelled: false, + id: ( + "client".to_string(), + "before_task_block_execute".to_string(), + ), + task_id, + instance_ids, + payloads, + }; + api.event_bus.fire(&mut event); + event + } + + pub fn check( + api: &ServerAPI, + task_id: String, + instance_ids: Vec, + payloads: Vec>>>, + ) -> bool { + Self::fire(api, task_id, instance_ids, payloads).cancelled + } +} diff --git a/enginelib/src/events/before_task_block_publish_event.rs b/enginelib/src/events/before_task_block_publish_event.rs new file mode 100644 index 0000000..c97cb64 --- /dev/null +++ b/enginelib/src/events/before_task_block_publish_event.rs @@ -0,0 +1,46 @@ +use std::sync::{Arc, RwLock}; + +use macros::Event; + +use crate::{Identifier, api::ServerAPI}; + +#[derive(Clone, Debug, Event)] +#[event(namespace = "client", name = "before_task_block_publish", cancellable)] +pub struct BeforeTaskBlockPublishEvent { + pub cancelled: bool, + pub id: Identifier, + pub task_id: String, + pub instance_ids: Vec, + pub payloads: Vec>>>, +} + +impl BeforeTaskBlockPublishEvent { + pub fn fire( + api: &ServerAPI, + task_id: String, + instance_ids: Vec, + payloads: Vec>>>, + ) -> Self { + let mut event = BeforeTaskBlockPublishEvent { + cancelled: false, + id: ( + "client".to_string(), + "before_task_block_publish".to_string(), + ), + task_id, + instance_ids, + payloads, + }; + api.event_bus.fire(&mut event); + event + } + + pub fn check( + api: &ServerAPI, + task_id: String, + instance_ids: Vec, + payloads: Vec>>>, + ) -> bool { + Self::fire(api, task_id, instance_ids, payloads).cancelled + } +} diff --git a/enginelib/src/events/before_task_execute_event.rs b/enginelib/src/events/before_task_execute_event.rs deleted file mode 100644 index 07e4d63..0000000 --- a/enginelib/src/events/before_task_execute_event.rs +++ /dev/null @@ -1,43 +0,0 @@ -use std::sync::{Arc, RwLock}; - -use macros::Event; - -use crate::{Identifier, api::ServerAPI}; - -#[derive(Clone, Debug, Event)] -#[event(namespace = "client", name = "before_task_execute", cancellable)] -pub struct BeforeTaskExecuteEvent { - pub cancelled: bool, - pub id: Identifier, - pub task_id: String, - pub instance_id: String, - pub payload: Arc>>, -} - -impl BeforeTaskExecuteEvent { - pub fn fire( - api: &ServerAPI, - task_id: String, - instance_id: String, - payload: Arc>>, - ) -> Self { - let mut event = BeforeTaskExecuteEvent { - cancelled: false, - id: ("client".to_string(), "before_task_execute".to_string()), - task_id, - instance_id, - payload, - }; - api.event_bus.fire(&mut event); - event - } - - pub fn check( - api: &ServerAPI, - task_id: String, - instance_id: String, - payload: Arc>>, - ) -> bool { - Self::fire(api, task_id, instance_id, payload).cancelled - } -} diff --git a/enginelib/src/events/before_task_publish_event.rs b/enginelib/src/events/before_task_publish_event.rs deleted file mode 100644 index f3c31dd..0000000 --- a/enginelib/src/events/before_task_publish_event.rs +++ /dev/null @@ -1,43 +0,0 @@ -use std::sync::{Arc, RwLock}; - -use macros::Event; - -use crate::{Identifier, api::ServerAPI}; - -#[derive(Clone, Debug, Event)] -#[event(namespace = "client", name = "before_task_publish", cancellable)] -pub struct BeforeTaskPublishEvent { - pub cancelled: bool, - pub id: Identifier, - pub task_id: String, - pub instance_id: String, - pub payload: Arc>>, -} - -impl BeforeTaskPublishEvent { - pub fn fire( - api: &ServerAPI, - task_id: String, - instance_id: String, - payload: Arc>>, - ) -> Self { - let mut event = BeforeTaskPublishEvent { - cancelled: false, - id: ("client".to_string(), "before_task_publish".to_string()), - task_id, - instance_id, - payload, - }; - api.event_bus.fire(&mut event); - event - } - - pub fn check( - api: &ServerAPI, - task_id: String, - instance_id: String, - payload: Arc>>, - ) -> bool { - Self::fire(api, task_id, instance_id, payload).cancelled - } -} diff --git a/enginelib/src/events/cgrpc_event.rs b/enginelib/src/events/cgrpc_event.rs index ef821a0..110768b 100644 --- a/enginelib/src/events/cgrpc_event.rs +++ b/enginelib/src/events/cgrpc_event.rs @@ -16,7 +16,7 @@ pub struct CgrpcEvent { impl CgrpcEvent { pub fn fire( - api: &mut ServerAPI, + api: &ServerAPI, handler_id: Identifier, payload: Vec, output: Arc>>, diff --git a/enginelib/src/events/mod.rs b/enginelib/src/events/mod.rs index 48ee7a5..ae15550 100644 --- a/enginelib/src/events/mod.rs +++ b/enginelib/src/events/mod.rs @@ -1,8 +1,8 @@ pub mod admin_auth_event; pub mod auth_event; -pub mod before_task_acquire_event; -pub mod before_task_execute_event; -pub mod before_task_publish_event; +pub mod before_task_block_acquire_event; +pub mod before_task_block_execute_event; +pub mod before_task_block_publish_event; pub mod cgrpc_event; pub mod client_auth_prepare_event; pub mod client_start_event; @@ -10,16 +10,14 @@ pub mod server_before_task_acquire_event; pub mod server_before_task_create_event; pub mod server_before_task_publish_event; pub mod server_start_event; -pub mod server_task_acquired_event; -pub mod server_task_created_event; -pub mod server_task_published_event; +pub mod server_task_block_acquired_event; +pub mod server_task_block_created_event; +pub mod server_task_block_published_event; pub mod start_event; -pub mod task_acquired_event; +pub mod task_block_acquired_event; use std::collections::HashMap; use std::sync::{Arc, RwLock}; -use sled::Db; - use crate::{Identifier, api::ServerAPI}; pub fn ID(namespace: &str, id: &str) -> Identifier { @@ -32,21 +30,21 @@ pub struct Events; impl Events { pub fn init_auth(_api: &mut ServerAPI) {} - pub fn CheckAuth(api: &mut ServerAPI, uid: String, challenge: String, db: Db) -> bool { + pub fn CheckAuth( + api: &ServerAPI, + uid: String, + challenge: String, + db: rust_rocksdb::DB, + ) -> bool { auth_event::AuthEvent::check(api, uid, challenge, db) } - pub fn CheckAdminAuth( - api: &mut ServerAPI, - payload: String, - target: Identifier, - db: Db, - ) -> bool { - admin_auth_event::AdminAuthEvent::check(api, payload, target, db) - } + // pub fn CheckAdminAuth(api: &ServerAPI, payload: String, target: Identifier, db: Db) -> bool { + // admin_auth_event::AdminAuthEvent::check(api, payload, target, db) + // } pub fn CgrpcEvent( - api: &mut ServerAPI, + api: &ServerAPI, handler_id: Identifier, payload: Vec, output: Arc>>, @@ -66,35 +64,50 @@ impl Events { client_auth_prepare_event::ClientAuthPrepareEvent::fire(api, headers) } - pub fn BeforeTaskAcquire(api: &ServerAPI, task_id: String) -> bool { - before_task_acquire_event::BeforeTaskAcquireEvent::check(api, task_id) + pub fn BeforeTaskBlockAcquire(api: &ServerAPI, task_ids: Vec) -> bool { + before_task_block_acquire_event::BeforeTaskBlockAcquireEvent::check(api, task_ids) } - pub fn TaskAcquired( + pub fn TaskBlockAcquired( api: &ServerAPI, task_id: String, - instance_id: String, - payload: Arc>>, + instance_ids: Vec, + payloads: Vec>>>, ) { - task_acquired_event::TaskAcquiredEvent::fire(api, task_id, instance_id, payload) + task_block_acquired_event::TaskBlockAcquiredEvent::fire( + api, + task_id, + instance_ids, + payloads, + ) } - pub fn BeforeTaskExecute( + pub fn BeforeTaskBlockExecute( api: &ServerAPI, task_id: String, - instance_id: String, - payload: Arc>>, + instance_ids: Vec, + payloads: Vec>>>, ) -> bool { - before_task_execute_event::BeforeTaskExecuteEvent::check(api, task_id, instance_id, payload) + before_task_block_execute_event::BeforeTaskBlockExecuteEvent::check( + api, + task_id, + instance_ids, + payloads, + ) } - pub fn BeforeTaskPublish( + pub fn BeforeTaskBlockPublish( api: &ServerAPI, task_id: String, - instance_id: String, - payload: Arc>>, + instance_ids: Vec, + payloads: Vec>>>, ) -> bool { - before_task_publish_event::BeforeTaskPublishEvent::check(api, task_id, instance_id, payload) + before_task_block_publish_event::BeforeTaskBlockPublishEvent::check( + api, + task_id, + instance_ids, + payloads, + ) } pub fn ServerStart(api: &ServerAPI) { @@ -109,21 +122,36 @@ impl Events { server_before_task_create_event::ServerBeforeTaskCreateEvent::check(api, task_id, payload) } - pub fn ServerTaskCreated( + pub fn ServerTaskBlockCreated( api: &ServerAPI, task_id: String, - instance_id: String, - payload: Arc>>, + instance_ids: Vec, + payloads: Vec>>>, ) { - server_task_created_event::ServerTaskCreatedEvent::fire(api, task_id, instance_id, payload) + server_task_block_created_event::ServerTaskBlockCreatedEvent::fire( + api, + task_id, + instance_ids, + payloads, + ) } pub fn ServerBeforeTaskAcquire(api: &ServerAPI, uid: String, task_id: String) -> bool { server_before_task_acquire_event::ServerBeforeTaskAcquireEvent::check(api, uid, task_id) } - pub fn ServerTaskAcquired(api: &ServerAPI, uid: String, task_id: String, instance_id: String) { - server_task_acquired_event::ServerTaskAcquiredEvent::fire(api, uid, task_id, instance_id) + pub fn ServerTaskBlockAcquired( + api: &ServerAPI, + uid: String, + task_id: String, + instance_ids: Vec, + ) { + server_task_block_acquired_event::ServerTaskBlockAcquiredEvent::fire( + api, + uid, + task_id, + instance_ids, + ) } pub fn ServerBeforeTaskPublish( @@ -142,7 +170,17 @@ impl Events { ) } - pub fn ServerTaskPublished(api: &ServerAPI, uid: String, task_id: String, instance_id: String) { - server_task_published_event::ServerTaskPublishedEvent::fire(api, uid, task_id, instance_id) + pub fn ServerTaskBlockPublished( + api: &ServerAPI, + uid: String, + task_id: String, + instance_ids: Vec, + ) { + server_task_block_published_event::ServerTaskBlockPublishedEvent::fire( + api, + uid, + task_id, + instance_ids, + ) } } diff --git a/enginelib/src/events/server_task_acquired_event.rs b/enginelib/src/events/server_task_block_acquired_event.rs similarity index 51% rename from enginelib/src/events/server_task_acquired_event.rs rename to enginelib/src/events/server_task_block_acquired_event.rs index 116ecf1..4d3f7ce 100644 --- a/enginelib/src/events/server_task_acquired_event.rs +++ b/enginelib/src/events/server_task_block_acquired_event.rs @@ -3,23 +3,23 @@ use macros::Event; use crate::{Identifier, api::ServerAPI}; #[derive(Clone, Debug, Event)] -#[event(namespace = "server", name = "task_acquired")] -pub struct ServerTaskAcquiredEvent { +#[event(namespace = "server", name = "task_block_acquired")] +pub struct ServerTaskBlockAcquiredEvent { pub cancelled: bool, pub id: Identifier, pub uid: String, pub task_id: String, - pub instance_id: String, + pub instance_ids: Vec, } -impl ServerTaskAcquiredEvent { - pub fn fire(api: &ServerAPI, uid: String, task_id: String, instance_id: String) { - let mut event = ServerTaskAcquiredEvent { +impl ServerTaskBlockAcquiredEvent { + pub fn fire(api: &ServerAPI, uid: String, task_id: String, instance_ids: Vec) { + let mut event = ServerTaskBlockAcquiredEvent { cancelled: false, - id: ("server".to_string(), "task_acquired".to_string()), + id: ("server".to_string(), "task_block_acquired".to_string()), uid, task_id, - instance_id, + instance_ids, }; api.event_bus.fire(&mut event); } diff --git a/enginelib/src/events/server_task_block_created_event.rs b/enginelib/src/events/server_task_block_created_event.rs new file mode 100644 index 0000000..9f5a331 --- /dev/null +++ b/enginelib/src/events/server_task_block_created_event.rs @@ -0,0 +1,33 @@ +use std::sync::{Arc, RwLock}; + +use macros::Event; + +use crate::{Identifier, api::ServerAPI}; + +#[derive(Clone, Debug, Event)] +#[event(namespace = "server", name = "task_block_created")] +pub struct ServerTaskBlockCreatedEvent { + pub cancelled: bool, + pub id: Identifier, + pub task_id: String, + pub instance_ids: Vec, + pub payloads: Vec>>>, +} + +impl ServerTaskBlockCreatedEvent { + pub fn fire( + api: &ServerAPI, + task_id: String, + instance_ids: Vec, + payloads: Vec>>>, + ) { + let mut event = ServerTaskBlockCreatedEvent { + cancelled: false, + id: ("server".to_string(), "task_block_created".to_string()), + task_id, + instance_ids, + payloads, + }; + api.event_bus.fire(&mut event); + } +} diff --git a/enginelib/src/events/server_task_published_event.rs b/enginelib/src/events/server_task_block_published_event.rs similarity index 51% rename from enginelib/src/events/server_task_published_event.rs rename to enginelib/src/events/server_task_block_published_event.rs index 5e6ee8a..43b997e 100644 --- a/enginelib/src/events/server_task_published_event.rs +++ b/enginelib/src/events/server_task_block_published_event.rs @@ -3,23 +3,23 @@ use macros::Event; use crate::{Identifier, api::ServerAPI}; #[derive(Clone, Debug, Event)] -#[event(namespace = "server", name = "task_published")] -pub struct ServerTaskPublishedEvent { +#[event(namespace = "server", name = "task_block_published")] +pub struct ServerTaskBlockPublishedEvent { pub cancelled: bool, pub id: Identifier, pub uid: String, pub task_id: String, - pub instance_id: String, + pub instance_ids: Vec, } -impl ServerTaskPublishedEvent { - pub fn fire(api: &ServerAPI, uid: String, task_id: String, instance_id: String) { - let mut event = ServerTaskPublishedEvent { +impl ServerTaskBlockPublishedEvent { + pub fn fire(api: &ServerAPI, uid: String, task_id: String, instance_ids: Vec) { + let mut event = ServerTaskBlockPublishedEvent { cancelled: false, - id: ("server".to_string(), "task_published".to_string()), + id: ("server".to_string(), "task_block_published".to_string()), uid, task_id, - instance_id, + instance_ids, }; api.event_bus.fire(&mut event); } diff --git a/enginelib/src/events/server_task_created_event.rs b/enginelib/src/events/server_task_created_event.rs deleted file mode 100644 index 1326818..0000000 --- a/enginelib/src/events/server_task_created_event.rs +++ /dev/null @@ -1,33 +0,0 @@ -use std::sync::{Arc, RwLock}; - -use macros::Event; - -use crate::{Identifier, api::ServerAPI}; - -#[derive(Clone, Debug, Event)] -#[event(namespace = "server", name = "task_created")] -pub struct ServerTaskCreatedEvent { - pub cancelled: bool, - pub id: Identifier, - pub task_id: String, - pub instance_id: String, - pub payload: Arc>>, -} - -impl ServerTaskCreatedEvent { - pub fn fire( - api: &ServerAPI, - task_id: String, - instance_id: String, - payload: Arc>>, - ) { - let mut event = ServerTaskCreatedEvent { - cancelled: false, - id: ("server".to_string(), "task_created".to_string()), - task_id, - instance_id, - payload, - }; - api.event_bus.fire(&mut event); - } -} diff --git a/enginelib/src/events/task_acquired_event.rs b/enginelib/src/events/task_acquired_event.rs deleted file mode 100644 index 7113cac..0000000 --- a/enginelib/src/events/task_acquired_event.rs +++ /dev/null @@ -1,33 +0,0 @@ -use std::sync::{Arc, RwLock}; - -use macros::Event; - -use crate::{Identifier, api::ServerAPI}; - -#[derive(Clone, Debug, Event)] -#[event(namespace = "client", name = "task_acquired")] -pub struct TaskAcquiredEvent { - pub cancelled: bool, - pub id: Identifier, - pub task_id: String, - pub instance_id: String, - pub payload: Arc>>, -} - -impl TaskAcquiredEvent { - pub fn fire( - api: &ServerAPI, - task_id: String, - instance_id: String, - payload: Arc>>, - ) { - let mut event = TaskAcquiredEvent { - cancelled: false, - id: ("client".to_string(), "task_acquired".to_string()), - task_id, - instance_id, - payload, - }; - api.event_bus.fire(&mut event); - } -} diff --git a/enginelib/src/events/task_block_acquired_event.rs b/enginelib/src/events/task_block_acquired_event.rs new file mode 100644 index 0000000..4b3be50 --- /dev/null +++ b/enginelib/src/events/task_block_acquired_event.rs @@ -0,0 +1,33 @@ +use std::sync::{Arc, RwLock}; + +use macros::Event; + +use crate::{Identifier, api::ServerAPI}; + +#[derive(Clone, Debug, Event)] +#[event(namespace = "client", name = "task_block_acquired")] +pub struct TaskBlockAcquiredEvent { + pub cancelled: bool, + pub id: Identifier, + pub task_id: String, + pub instance_ids: Vec, + pub payloads: Vec>>>, +} + +impl TaskBlockAcquiredEvent { + pub fn fire( + api: &ServerAPI, + task_id: String, + instance_ids: Vec, + payloads: Vec>>>, + ) { + let mut event = TaskBlockAcquiredEvent { + cancelled: false, + id: ("client".to_string(), "task_block_acquired".to_string()), + task_id, + instance_ids, + payloads, + }; + api.event_bus.fire(&mut event); + } +} diff --git a/enginelib/src/lib.rs b/enginelib/src/lib.rs index 7fc70e0..6134f59 100644 --- a/enginelib/src/lib.rs +++ b/enginelib/src/lib.rs @@ -3,6 +3,7 @@ pub mod api; pub mod config; pub mod event; pub mod events; +pub mod nucleus; extern crate self as enginelib; pub use inventory; pub mod plugin; @@ -17,3 +18,4 @@ pub trait Registry: Default + Clone { fn get(&self, identifier: &Identifier) -> Option>; } pub use chrono; +mod error; diff --git a/enginelib/src/nucleus/cancel.rs b/enginelib/src/nucleus/cancel.rs new file mode 100644 index 0000000..c731a8e --- /dev/null +++ b/enginelib/src/nucleus/cancel.rs @@ -0,0 +1,5 @@ +use std::sync::Arc; + +use crate::api::ServerAPI; + +fn cancel(api: Arc) {} diff --git a/enginelib/src/nucleus/complete.rs b/enginelib/src/nucleus/complete.rs new file mode 100644 index 0000000..be6c144 --- /dev/null +++ b/enginelib/src/nucleus/complete.rs @@ -0,0 +1,5 @@ +use std::sync::Arc; + +use crate::api::ServerAPI; + +fn complete(api: Arc) {} diff --git a/enginelib/src/nucleus/lease.rs b/enginelib/src/nucleus/lease.rs new file mode 100644 index 0000000..c036050 --- /dev/null +++ b/enginelib/src/nucleus/lease.rs @@ -0,0 +1,5 @@ +use std::sync::Arc; + +use crate::api::ServerAPI; + +fn lease(api: Arc) {} diff --git a/enginelib/src/nucleus/mod.rs b/enginelib/src/nucleus/mod.rs new file mode 100644 index 0000000..600e8bb --- /dev/null +++ b/enginelib/src/nucleus/mod.rs @@ -0,0 +1,6 @@ +mod cancel; +mod complete; +mod lease; +mod query; +mod renew; +mod submit; diff --git a/enginelib/src/nucleus/query.rs b/enginelib/src/nucleus/query.rs new file mode 100644 index 0000000..8e68e50 --- /dev/null +++ b/enginelib/src/nucleus/query.rs @@ -0,0 +1,5 @@ +use std::sync::Arc; + +use crate::api::ServerAPI; + +fn query(api: Arc) {} diff --git a/enginelib/src/nucleus/renew.rs b/enginelib/src/nucleus/renew.rs new file mode 100644 index 0000000..0161a7b --- /dev/null +++ b/enginelib/src/nucleus/renew.rs @@ -0,0 +1,5 @@ +use std::sync::Arc; + +use crate::api::ServerAPI; + +fn renew(api: Arc) {} diff --git a/enginelib/src/nucleus/submit.rs b/enginelib/src/nucleus/submit.rs new file mode 100644 index 0000000..099a92a --- /dev/null +++ b/enginelib/src/nucleus/submit.rs @@ -0,0 +1,34 @@ +use std::sync::Arc; + +use crate::{ + Identifier, Registry, + api::ServerAPI, + error::{Error, ErrorKind}, + task::Task, +}; +// t:namespace:task_name: -> Serialized Task Record +// f:namespace:task_name: -> Finished Task Record +pub fn submit(api: Arc, task_bytes: &[u8], task_id: Identifier) -> Result<(), Error> { + // deserialize + + let task = api + .task_registry + .get(&task_id) + .ok_or(Error::new("Not found".into()))?; + if task.verify(task_bytes) { + let res = api.db.put( + format!( + "t:{}:{}:{}", + task_id.0, + task_id.1, + druid::Druid::default().to_hex() + ), + task_bytes, + ); + if res.is_err() { + return Err(Error::new(res.err().unwrap().to_string())); + } + } + + Ok(()) +} diff --git a/enginelib/src/plugin.rs b/enginelib/src/plugin.rs index d9fe447..8847b3d 100644 --- a/enginelib/src/plugin.rs +++ b/enginelib/src/plugin.rs @@ -64,7 +64,8 @@ impl LibraryManager { drop(self); } - pub fn load_modules(&mut self, api: &mut ServerAPI) { + pub fn load_modules(api: &mut ServerAPI) { + let mut library_manager = LibraryManager::default(); let dir_path = "./mods"; let mut files: Vec = Vec::new(); @@ -92,8 +93,9 @@ impl LibraryManager { info!("Found {} module(s) to load", files.len()); for file in files { - self.load_module(&file, api); + library_manager.load_module(&file, api); } + api.lib_manager = library_manager; } pub fn load_module(&mut self, path: &str, api: &mut ServerAPI) { diff --git a/enginelib/src/prelude.rs b/enginelib/src/prelude.rs index bee62a5..8e84693 100644 --- a/enginelib/src/prelude.rs +++ b/enginelib/src/prelude.rs @@ -10,11 +10,11 @@ pub use crate::event::{ Event, EventBus, EventCTX, EventHandler, EventRegistrar, register_inventory_handlers, register_inventory_handlers_for_origin, }; -pub use crate::events::admin_auth_event::AdminAuthEvent; +//pub use crate::events::admin_auth_event::AdminAuthEvent; pub use crate::events::auth_event::AuthEvent; -pub use crate::events::before_task_acquire_event::BeforeTaskAcquireEvent; -pub use crate::events::before_task_execute_event::BeforeTaskExecuteEvent; -pub use crate::events::before_task_publish_event::BeforeTaskPublishEvent; +pub use crate::events::before_task_block_acquire_event::BeforeTaskBlockAcquireEvent; +pub use crate::events::before_task_block_execute_event::BeforeTaskBlockExecuteEvent; +pub use crate::events::before_task_block_publish_event::BeforeTaskBlockPublishEvent; pub use crate::events::cgrpc_event::CgrpcEvent; pub use crate::events::client_auth_prepare_event::ClientAuthPrepareEvent; pub use crate::events::client_start_event::ClientStartEvent; @@ -22,11 +22,11 @@ pub use crate::events::server_before_task_acquire_event::ServerBeforeTaskAcquire pub use crate::events::server_before_task_create_event::ServerBeforeTaskCreateEvent; pub use crate::events::server_before_task_publish_event::ServerBeforeTaskPublishEvent; pub use crate::events::server_start_event::ServerStartEvent; -pub use crate::events::server_task_acquired_event::ServerTaskAcquiredEvent; -pub use crate::events::server_task_created_event::ServerTaskCreatedEvent; -pub use crate::events::server_task_published_event::ServerTaskPublishedEvent; +pub use crate::events::server_task_block_acquired_event::ServerTaskBlockAcquiredEvent; +pub use crate::events::server_task_block_created_event::ServerTaskBlockCreatedEvent; +pub use crate::events::server_task_block_published_event::ServerTaskBlockPublishedEvent; pub use crate::events::start_event::StartEvent; -pub use crate::events::task_acquired_event::TaskAcquiredEvent; +pub use crate::events::task_block_acquired_event::TaskBlockAcquiredEvent; pub use crate::events::{Events, ID}; pub use crate::plugin::{LibraryDependency, LibraryMetadata}; pub use crate::task::{Runner, Task, Verifiable}; diff --git a/enginelib/src/task.rs b/enginelib/src/task.rs index 410a90f..0798acf 100644 --- a/enginelib/src/task.rs +++ b/enginelib/src/task.rs @@ -1,35 +1,14 @@ use std::fmt::Debug; -use std::{collections::HashMap, sync::Arc}; +use std::sync::Arc; use crate::Identifier; use chrono::{DateTime, Utc}; -use crossbeam::queue::ArrayQueue; +use dashmap::DashMap; use serde::{Deserialize, Serialize}; use tracing::{error, instrument, warn}; -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct StoredTask { - pub bytes: Vec, - pub id: String, -} -#[derive(Debug, Default, Clone, Serialize, Deserialize)] -pub struct LeasedTask { - pub stored_task: Arc, - pub user_id: String, - pub given_at: DateTime, -} -#[derive(Debug, Default)] -pub struct TaskQueue { - pub tasks: HashMap>>, -} - -#[derive(Debug, Default, Clone)] -pub struct LeasedTaskQueue { - pub tasks: HashMap>, -} - pub trait Verifiable { - fn verify(&self, b: Vec) -> bool; + fn verify(&self, b: &[u8]) -> bool; } pub trait Task: Debug + Sync + Send + Verifiable { fn get_id(&self) -> Identifier; diff --git a/enginelib/tests/event_tests.rs b/enginelib/tests/event_tests.rs deleted file mode 100644 index ed5ec9a..0000000 --- a/enginelib/tests/event_tests.rs +++ /dev/null @@ -1,221 +0,0 @@ -use enginelib::{Registry, api::EngineAPI, events::ID, task::Verifiable}; -use macros::Verifiable; -use std::sync::Arc; -use tracing_test::traced_test; - -#[derive(Clone, Debug, macros::Event)] -#[event(namespace = "test", name = "test_event", cancellable)] -struct TestEvent { - pub value: i32, - pub cancelled: bool, -} - -#[macros::event_handler(namespace = "test", name = "test_event")] -fn increment_test_event(event: &mut TestEvent) { - event.value += 1; -} - -#[derive(Clone, Debug, macros::Event)] -#[event(namespace = "test", name = "stateful_event")] -struct StatefulEvent { - pub value: u32, -} - -#[macros::event_handler(namespace = "test", name = "stateful_event", ctx = 7u32)] -fn add_ctx_to_event(event: &mut StatefulEvent, ctx: &u32) { - event.value += *ctx; -} - -#[traced_test] -#[test] -fn id() { - assert!(ID("namespace", "id") == ID("namespace", "id")) -} - -#[traced_test] -#[test] -fn test_event_registration_and_handling() { - let mut api = EngineAPI::test_default(); - - let mut test_event = TestEvent { - value: 0, - cancelled: false, - }; - - enginelib::event::register_inventory_handlers(&mut api); - api.event_bus.fire(&mut test_event); - assert_eq!(test_event.value, 1); - drop(api.db); -} - -#[traced_test] -#[test] -fn test_stateful_event_auto_registration() { - let mut api = EngineAPI::test_default(); - - enginelib::event::register_inventory_handlers(&mut api); - - let mut event = StatefulEvent { value: 0 }; - api.event_bus.fire(&mut event); - assert_eq!(event.value, 7); - drop(api.db); -} - -#[traced_test] -#[test] -fn test_task_registration() { - use enginelib::task::Task; - use serde::{Deserialize, Serialize}; - - #[derive(Debug, Clone, Serialize, Deserialize, Verifiable)] - struct TestTask { - pub value: i32, - pub id: (String, String), - } - - impl Task for TestTask { - fn to_toml(&self) -> String { - "".into() - } - fn get_id(&self) -> (String, String) { - self.id.clone() - } - fn clone_box(&self) -> Box { - Box::new(self.clone()) - } - fn run_cpu(&mut self) { - self.value += 1; - } - fn to_bytes(&self) -> Vec { - postcard::to_allocvec(self).unwrap() - } - fn from_bytes(&self, bytes: &[u8]) -> Box { - Box::new(postcard::from_bytes::(bytes).unwrap()) - } - fn from_toml(&self, d: String) -> Box { - return Box::new(self.clone()); - } - } - let mut api = EngineAPI::test_default(); - let task_id = ID("test", "test_task"); - - // Register the task type - api.task_registry.register( - Arc::new(TestTask { - value: 0, - id: task_id.clone(), - }), - task_id.clone(), - ); - - // Verify it was registered - assert!(api.task_registry.tasks.contains_key(&task_id)); - drop(api.db); -} - -#[traced_test] -#[test] -fn test_task_execution() { - use enginelib::task::{Runner, Task}; - use serde::{Deserialize, Serialize}; - - #[derive(Debug, Clone, Serialize, Deserialize, Verifiable)] - struct TestTask { - pub value: i32, - pub id: (String, String), - } - - impl Task for TestTask { - fn to_toml(&self) -> String { - "".into() - } - fn from_toml(&self, d: String) -> Box { - return Box::new(self.clone()); - } - fn get_id(&self) -> (String, String) { - self.id.clone() - } - fn clone_box(&self) -> Box { - Box::new(self.clone()) - } - fn run_cpu(&mut self) { - self.value += 1; - } - fn to_bytes(&self) -> Vec { - postcard::to_allocvec(self).unwrap() - } - fn from_bytes(&self, bytes: &[u8]) -> Box { - Box::new(postcard::from_bytes::(bytes).unwrap()) - } - } - - // Test task execution directly - let mut task = TestTask { - value: 0, - id: ID("test", "test_task"), - }; - - task.run(Some(Runner::CPU)); - assert_eq!(task.value, 1); -} - -#[traced_test] -#[test] -fn test_task_serialization() { - use enginelib::task::{StoredTask, Task}; - use serde::{Deserialize, Serialize}; - - #[derive(Debug, Clone, Serialize, Deserialize, Verifiable)] - struct TestTask { - pub value: i32, - pub id: (String, String), - } - - impl Task for TestTask { - fn to_toml(&self) -> String { - "".into() - } - fn from_toml(&self, d: String) -> Box { - return Box::new(self.clone()); - } - fn get_id(&self) -> (String, String) { - self.id.clone() - } - fn clone_box(&self) -> Box { - Box::new(self.clone()) - } - fn run_cpu(&mut self) { - self.value += 1; - } - fn to_bytes(&self) -> Vec { - postcard::to_allocvec(self).unwrap() - } - fn from_bytes(&self, bytes: &[u8]) -> Box { - Box::new(postcard::from_bytes::(bytes).unwrap()) - } - } - - let task = TestTask { - value: 42, - id: ID("test", "test_task"), - }; - - // Test serialization and deserialization - let serialized = task.to_bytes(); - let stored_task = StoredTask { - bytes: serialized, - id: "id".into(), - }; - - // Deserialize - let deserialized_task: TestTask = postcard::from_bytes(&stored_task.bytes).unwrap(); - assert_eq!(deserialized_task.value, 42); - - // Test the from_bytes function - let recreated_task = task.from_bytes(&stored_task.bytes); - // We need a way to check the value inside the recreated task - // Since we can't directly access the value, we'll serialize it again and deserialize manually - let bytes = recreated_task.to_bytes(); - let final_task: TestTask = postcard::from_bytes(&bytes).unwrap(); - assert_eq!(final_task.value, 42); -} diff --git a/rfc/rfc1006.md b/rfc/rfc1006.md new file mode 100644 index 0000000..834ed28 --- /dev/null +++ b/rfc/rfc1006.md @@ -0,0 +1,1176 @@ +# RFC 1006: Three-Engine Server Architecture + +- **Status:** Draft +- **Scope:** Near-total rewrite of the server runtime +- **Audience:** Engine maintainers, transport authors, data-backend authors, and mod authors +- **Last updated:** 2026-07-12 + +## Summary + +Replace the current server structure with three strict architectural engines: + +1. **COM Engine** — accepts requests over a communication technology such as gRPC, HTTP, a message broker, an embedded API, or a future protocol; translates between that protocol and transport-neutral server commands; applies protocol-level flow control; and maps server errors back to protocol errors. +2. **Server Engine** — implements authentication, authorization, task use cases, task validation, lifecycle policy, mod hooks, and orchestration. It contains no `tonic`, protobuf, sled, socket, or database-key knowledge. +3. **Data Engine** — owns durable task state, atomic task transitions, leasing, idempotency, pagination, and backend-specific acceleration. It contains no gRPC or server-policy knowledge. + +This is intentionally a near-rewrite. The desired result is not `EngineService` hidden behind one trait. The desired result is a server whose application semantics can run without gRPC and without sled, and whose gRPC and sled implementations can be replaced independently. + +The three engines are **logical boundaries and Rust crates**, not three mandatory network services. The first production implementation SHOULD compose them in one process. A later deployment MAY move COM or Data across a process boundary, but only after a measured need and a separately versioned internal protocol exist. + +## Decision in one diagram + +```text + external clients + | + | gRPC / HTTP / broker / embedded calls + v ++-----------------------+ +| COM Engine | protocol parsing, framing, limits, +| grpc | http | ... | context extraction, error mapping ++-----------+-----------+ + | InboundPort: transport-neutral commands/results + v ++-----------------------+ +| Server Engine | authn/authz, validation, lifecycle, +| application use cases | policies, mods, orchestration ++-----------+-----------+ + | DataPort: atomic domain operations + v ++-----------------------+ +| Data Engine | durable state machine, leases, +| sled | postgres | mem | idempotency, cursors, transactions ++-----------------------+ +``` + +Dependencies point inward: + +```text +engine-com-grpc ----> engine-server-api <---- engine-server + ^ | + | v + engine-domain <------ engine-data-api + ^ + | + engine-data-sled / engine-data-memory / future adapters +``` + +No arrow in the architecture may point from `engine-server` to a COM implementation or from a domain type to a storage implementation. + +## Terminology and normative language + +The words **MUST**, **MUST NOT**, **SHOULD**, **SHOULD NOT**, and **MAY** describe requirements for the rewrite. + +- **Task type**: the registered `(namespace, task)` pair, currently represented by `Identifier`. +- **Task instance**: one submitted unit of work with its own opaque ID. +- **Lease**: a time-bounded, exclusive claim to attempt a task. +- **Completion**: an accepted result for the current lease. +- **Engine**: a cohesive logical component with a public contract. It does not imply a process. +- **Port**: an application-owned contract. +- **Adapter**: a technology-specific implementation of a port. +- **Command**: a request to perform an application use case. + +## Motivation + +The current code is optimized around direct access to a shared `ServerAPI`. That made early development fast, but it makes communication, application policy, concurrency, and persistence change together. + +The gRPC service in `engine/src/lib.rs` currently does all of the following inside request handlers: + +- reads tonic metadata and constructs identity/authentication inputs; +- parses protobuf-shaped task identifiers; +- invokes auth and lifecycle events; +- queries the task registry and validates payloads; +- reads queue configuration; +- refills in-memory channels by scanning sled; +- creates and removes in-memory leases; +- performs sled batches and maps sled failures to tonic statuses; +- implements batching and stream framing; +- chooses which malformed or invalid batch members are silently skipped; +- constructs protobuf responses. + +`enginelib/src/api.rs` combines configuration, queue channels, lease maps, refill cursors, ID generation, task registration, event dispatch, module loading, sled access, logging setup, and background lease cleanup in one public struct. Mods receive that struct directly through the native Rust ABI. + +This has concrete consequences: + +| Current condition | Consequence | +|---|---| +| `EngineService` accepts `tonic::Request` in internal helpers | A non-gRPC adapter must fabricate tonic values or duplicate the use case. | +| Domain operations return `tonic::Status` | Application errors cannot be reused or mapped consistently by another protocol. | +| `ServerAPI` exposes `sled::Db` and queue internals | Storage cannot be replaced without changing server logic, auth events, mods, and tests. | +| Leases live in RAM while queued payloads live in sled | Restart semantics are implicit; lease ownership and persistence are not one atomic transition. | +| Queue refill, active IDs, and leases coordinate across several maps | Correctness relies on ordering spread across COM-facing code and data code. | +| Streaming RPCs buffer or batch through unary internal helpers | Transport framing has accidentally become application behavior. | +| Batch members are sometimes skipped while the overall RPC succeeds | Clients cannot reliably distinguish accepted, rejected, and retriable items. | +| Mods receive `&mut ServerAPI` via `extern "Rust"` | Every field-level redesign risks ABI/API breakage and grants mods unrestricted authority. | +| Page-number pagination scans and materializes state | Cost grows with the dataset and exposes backend behavior through the API. | + +The rewrite MUST make task lifecycle rules explicit before pursuing additional throughput optimizations. A faster implementation of ambiguous state transitions is still ambiguous. + +## Goals + +1. Replace gRPC without changing Server Engine or Data Engine code. +2. Replace sled without changing Server Engine or any COM adapter. +3. Run the complete application use cases in tests without sockets or a production database. +4. Define one authoritative task state machine with atomic transitions. +5. Make retry, idempotency, deadlines, cancellation, partial batch success, and delivery guarantees explicit. +6. Preserve high-throughput block operations without making block size part of correctness. +7. Narrow mod capabilities so mods do not depend on a god object or database handle. +8. Support gradual migration and rollback while the current gRPC API remains available. +9. Make process boundaries optional deployment choices rather than accidental code dependencies. +10. Give humans enough contracts, invariants, and acceptance criteria to implement the rewrite without rediscovering the architecture. + +## Non-goals + +- This RFC does not require three deployable services in the first release. +- This RFC does not select the final production database. It defines the contract a backend must satisfy. +- This RFC does not promise exactly-once task execution. It defines at-least-once leasing with idempotent completion. +- This RFC does not preserve source compatibility for mods that directly access `ServerAPI`. +- This RFC does not require keeping the current proto naming mistakes in a new API. The legacy adapter may preserve them. +- This RFC does not define worker-side execution architecture beyond the server-facing protocol. +- This RFC does not introduce arbitrary generic repositories for every struct. Ports are use-case and state-transition oriented. + +## Architectural principles + +### 1. Ports describe purposeful conversations + +The architecture follows the ports-and-adapters idea: technology-specific adapters translate an external interaction into an application port. A port is not “all database methods” or “all gRPC methods”; it is a coherent conversation such as submit, lease, complete, inspect, or administer tasks. + +### 2. Logical separation comes before process separation + +COM, Server, and Data MUST compile as independent boundaries. They SHOULD initially run in one process: + +- calls stay cheap and typed; +- cancellation and backpressure do not require another wire protocol; +- deployment remains simple; +- benchmarks reveal real boundary cost; +- a later split is possible because dependencies are already controlled. + +Turning each engine into a service immediately is rejected by this RFC. It would create an internal COM-to-Server protocol and possibly a Server-to-Data protocol, add new partial-failure modes, and make a communication-replacement project paradoxically depend on more communication. + +### 3. The Data Engine owns state transitions, not CRUD + +The Server Engine MUST NOT implement “read state, decide, then write state” for lifecycle transitions. The Data Engine MUST expose atomic operations such as `lease`, `complete`, `renew`, and `release_expired`. + +A generic `get_task` plus `put_task` repository is insufficient because it allows races to be reconstructed above the transaction boundary. + +### 4. The durable store is authoritative + +In-memory queues, indexes, caches, cursors, and channels MAY accelerate selection, but MUST NOT be the only record of: + +- task existence; +- current lifecycle state; +- lease holder/token and lease deadline; +- accepted completion; +- idempotency outcome. + +After a crash, correctness MUST be recoverable from durable data without reconstructing hidden facts from logs. + +### 5. External schemas are adapters, not domain models + +Generated protobuf types MUST remain inside `engine-com-grpc`. JSON, broker, or future protocol types belong to their adapters. The domain owns explicit Rust value types. Mapping code is intentional and tested. + +### 6. Errors are domain facts before protocol statuses + +The Server Engine returns a closed, transport-neutral error taxonomy. Each COM adapter maps it to its own error mechanism. A storage error is never formatted directly into a public response. + +### 7. Correctness semantics precede optimization + +Batching, caching, lock sharding, and zero-copy work may optimize a contract but MUST NOT redefine it. Every optimized Data Engine implementation must pass the same contract suite as the reference in-memory implementation. + +## Proposed workspace + +Names can change during implementation, but the dependency boundaries may not. + +| Crate | Responsibility | Allowed major dependencies | Forbidden knowledge | +|---|---|---|---| +| `engine-domain` | IDs, task values, state, commands/results, errors, invariants | `serde`, time/ID libraries if needed | tonic, prost, sled, sockets, config files | +| `engine-server-api` | inbound application port and request context | `engine-domain` | protobuf and database types | +| `engine-server` | use cases, authz, validation, policy, event orchestration | domain, server API, data API, mod host API | tonic, prost, sled keys | +| `engine-data-api` | atomic outbound data port and backend contract tests | domain | tonic, external protocol DTOs | +| `engine-data-memory` | deterministic reference/test adapter | data API | transport code | +| `engine-data-sled` | migration/compatibility backend | sled, data API | tonic, auth policy, mods | +| `engine-com-grpc` | legacy and/or v2 gRPC adapters | tonic, prost, server API | sled, queue maps, task validators | +| `engine-runtime` | configuration, dependency injection, startup, shutdown, health | all selected adapters | business rules | +| `engine-mod-api` | stable, capability-limited mod contracts | domain-safe DTOs | `ServerRuntime`, raw DB handles | +| `engine-client` | worker/client behavior | generated client adapter or protocol SDK | server persistence internals | + +The existing `engine` package MAY temporarily contain compatibility binaries, but the final `server` binary should be a thin composition root in `engine-runtime`. + +### Compile-time enforcement + +The workspace SHOULD add automated dependency checks. At minimum: + +- `engine-domain` has no dependency on any engine implementation crate; +- `engine-server` has no dependency on `tonic`, `prost`, `sled`, or a COM crate; +- COM crates have no dependency on Data implementations; +- Data crates have no dependency on Server or COM implementations; +- generated protobuf modules are not re-exported by domain/server crates. + +These constraints SHOULD be checked in CI, not left to review convention. + +## Domain model + +Stringly typed identifiers and overloaded payload fields should be replaced with explicit types. Example shapes are illustrative, not final syntax: + +```rust +pub struct TaskType { + pub namespace: Namespace, + pub name: TaskName, +} + +pub struct TaskId(pub Uuid); +pub struct LeaseId(pub Uuid); +pub struct IdempotencyKey(pub String); +pub struct PrincipalId(pub String); + +pub struct TaskInput(pub bytes::Bytes); +pub struct TaskOutput(pub bytes::Bytes); + +pub enum TaskState { + Queued, + Leased, + Completed, + Cancelled, + DeadLettered, +} +``` + +### ID rules + +- IDs MUST be generated by an injected `IdGenerator`, not by wall-clock formatting inside storage code. +- IDs MUST be globally unique across server processes if horizontal Server Engine instances are allowed. +- IDs MUST be opaque in public contracts; clients may compare or store them but must not parse routing data from them. +- UUIDv7 is RECOMMENDED because it is standardized, globally usable, and time-orderable. A different 128-bit opaque scheme is acceptable if collision and ordering properties are documented. +- Lease IDs MUST be unguessable and distinct for each acquisition attempt. + +### Task record + +The conceptual durable record contains at least: + +```text +task_id +task_type(namespace, name) +input_bytes +output_bytes? // or an output reference for large results +state +created_at +updated_at +available_at +priority +attempt +max_attempts? +lease_id? +lease_owner? +lease_deadline? +completed_at? +input_digest +output_digest? +revision // monotonically changing fencing/version value +``` + +Backend schemas may normalize or partition this representation. Observable semantics must remain the same. + +### State machine + +```text + lease expires / release + +-----------------------------+ + | | + v | + submit ------> QUEUED -------- lease --------> LEASED + | | + | cancel | complete(current lease) + v v + CANCELLED COMPLETED + + LEASED -- max attempts / policy --> DEAD_LETTERED + QUEUED -- max age / admin policy -> DEAD_LETTERED +``` + +Required invariants: + +1. A task has exactly one authoritative state. +2. A successful lease transition returns a unique `LeaseId`, increments `attempt` and `revision`, and durably records its deadline in the same atomic operation. +3. At most one unexpired lease is current for a task. +4. Only the current `LeaseId` may renew, release, or complete the task. +5. Completion atomically checks the lease token and changes `LEASED -> COMPLETED`. +6. A stale worker cannot complete a task after its lease expired and another worker acquired it. The lease ID/revision is a fencing token. +7. Repeating a successful completion with the same task ID, lease ID, and output digest returns the original success. +8. Repeating it with different output returns `Conflict`. +9. An expired lease is available for another attempt even if cleanup has not physically rewritten the row yet, provided the backend can atomically claim it. +10. Deletion MUST NOT make a concurrently leased task appear successfully completed. Administrative deletion semantics must be explicit: reject, cancel/tombstone, or force-delete with audit. + +### Delivery guarantee + +The engine promises **at-least-once delivery**: + +- a worker may receive a task and fail before completion; +- a response may be lost after a commit; +- an expired task may be leased again; +- therefore task execution may occur more than once. + +The engine does not claim exactly-once execution. It provides fencing and idempotent completion so only a valid completion is accepted. Task implementations that cause external side effects remain responsible for their own idempotency or transactional integration. + +## Server Engine + +The Server Engine owns application behavior. It is invoked through an inbound port and depends only on outbound ports. + +### Inbound request context + +Every COM adapter constructs a bounded context: + +```rust +pub struct RequestContext { + pub request_id: RequestId, + pub credential: Credential, + pub deadline: Option, + pub cancellation: Cancellation, + pub trace: TraceContext, + pub peer: Option, +} +``` + +Rules: + +- `credential` is opaque to COM except for syntax/size checks. +- Authentication and authorization are Server Engine policies through an `AuthPort`. +- The context MUST NOT expose tonic metadata maps to the Server Engine. +- Deadlines and cancellation MUST be checked before expensive work and passed to outbound operations. +- Credentials MUST NOT be copied into tracing baggage, logs, errors, or mod events. +- `request_id` is for correlation, not idempotency. Mutating commands use an explicit idempotency key. + +### Inbound port + +The public shape should be use-case oriented: + +```rust +pub trait ServerPort: Send + Sync { + async fn metadata(&self, ctx: RequestContext) -> Result; + async fn list_task_types(&self, ctx: RequestContext) -> Result, EngineError>; + async fn submit(&self, ctx: RequestContext, cmd: SubmitTasks) + -> Result; + async fn lease(&self, ctx: RequestContext, cmd: LeaseTasks) + -> Result; + async fn complete(&self, ctx: RequestContext, cmd: CompleteTasks) + -> Result; + async fn renew(&self, ctx: RequestContext, cmd: RenewLeases) + -> Result; + async fn query(&self, ctx: RequestContext, query: TaskQuery) + -> Result; + async fn administer(&self, ctx: RequestContext, cmd: AdminCommand) + -> Result; + async fn invoke_extension(&self, ctx: RequestContext, cmd: ExtensionCommand) + -> Result; +} +``` + +Implementation note: the exact Rust async-trait representation must be chosen deliberately. Native `async fn` in traits and `dyn` dispatch constraints evolve independently. The project may use generic composition, boxed futures, or a small type-erasure crate. This is an implementation choice, not a reason to leak adapters inward. + +### Use-case pipeline + +Every mutating use case follows the same conceptual sequence: + +```text +validate structural limits + -> authenticate + -> authorize action/resource + -> run cancellable before-policy/mod hooks + -> validate task-type-specific content + -> invoke one atomic Data Engine operation + -> emit post-commit domain events + -> return per-item result +``` + +Ordering deviations MUST be documented. In particular, payload validation must occur before durable submission, while lease-token validation belongs inside the atomic completion transition. + +### Batch semantics + +Blocks remain important for throughput, but vague partial success is prohibited. + +Every batch command MUST declare one of two modes: + +- `Atomic`: all valid items commit together or none commit. A structurally invalid member rejects the whole batch. +- `BestEffort`: each member returns an explicit result, and successful items may commit independently or in backend sub-batches. + +Example result: + +```rust +pub struct BatchResult { + pub items: Vec>, +} + +pub enum ItemResult { + Applied(T), + AlreadyApplied(T), + Rejected(ItemError), + Retryable(ItemError), +} +``` + +The COM adapter may choose a protocol representation, but it MUST NOT turn rejected members into silent overall success. + +### Task validation registry + +The Server Engine owns a `TaskCatalog` port: + +```rust +pub trait TaskCatalog: Send + Sync { + fn descriptor(&self, task_type: &TaskType) -> Option; + fn validate_input(&self, task_type: &TaskType, input: &[u8]) -> ValidationResult; + fn validate_output(&self, task_type: &TaskType, output: &[u8]) -> ValidationResult; +} +``` + +Data does not call mod code or understand task payloads. COM does not validate domain payloads. The catalog is built during startup and becomes immutable or versioned for the life of requests. + +### Error model + +The Server Engine MUST return structured errors with stable codes: + +```rust +pub enum EngineErrorCode { + InvalidArgument, + Unauthenticated, + PermissionDenied, + NotFound, + AlreadyExists, + Conflict, + FailedPrecondition, + ResourceExhausted, + Cancelled, + DeadlineExceeded, + Unavailable, + Internal, +} + +pub struct EngineError { + pub code: EngineErrorCode, + pub safe_message: String, + pub retry: RetryAdvice, + pub details: Vec, + pub error_id: ErrorId, +} +``` + +- `safe_message` is client-visible and MUST contain no database keys, credentials, filesystem paths, panic text, or internal module details. +- Internal sources are retained in logs/spans keyed by `error_id`. +- Retryability is explicit and may include a minimum delay. +- Adapters own protocol mapping; the domain never imports protocol status types. + +## Data Engine + +The Data Engine contract is the most important rewrite boundary because it centralizes lifecycle correctness. + +### Port shape + +The port MUST expose transitions rather than storage primitives: + +```rust +pub trait DataPort: Send + Sync { + async fn submit(&self, op: SubmitOp) -> Result; + async fn lease(&self, op: LeaseOp) -> Result, DataError>; + async fn complete(&self, op: CompleteOp) -> Result; + async fn renew(&self, op: RenewOp) -> Result; + async fn release(&self, op: ReleaseOp) -> Result; + async fn query(&self, op: QueryOp) -> Result, DataError>; + async fn administer(&self, op: DataAdminOp) -> Result; + async fn health(&self) -> DataHealth; +} +``` + +The port MUST NOT expose: + +- raw sled/database handles; +- backend key construction; +- iterators whose lifetime is tied to backend guards; +- channels or `DashMap` references; +- a transaction object that lets the Server Engine perform arbitrary reads/writes; +- protocol response types. + +### Atomic operations + +#### Submit + +`submit` atomically: + +1. checks the idempotency key in the command scope; +2. allocates or accepts Server-generated task IDs according to one documented rule; +3. inserts queued tasks; +4. records the idempotency result; +5. makes tasks visible to lease selection only after commit. + +If the client retries after a lost response, the same idempotency key and same command digest return the same task IDs. The same key with a different digest returns `Conflict`. + +#### Lease + +`lease` atomically selects up to `limit` eligible tasks and changes their state to leased. It records the owner, unique lease ID, attempt number, deadline, and revision before returning payloads. + +Selection MAY consider task type, priority, `available_at`, and fairness. Ordering is a documented policy; it is not inferred from backend key order. + +The lease duration has server-configured minimum and maximum bounds. A client-provided value outside those bounds is rejected or clamped according to one documented API rule. + +#### Complete + +`complete` atomically: + +1. finds the task; +2. verifies that it is leased; +3. verifies the current lease ID and revision; +4. verifies that completion is allowed at the transaction's notion of time; +5. applies idempotency rules; +6. stores output or its reference; +7. transitions to completed; +8. records a post-commit event/outbox item if configured. + +The Server Engine validates output content before this call, but the Data Engine enforces ownership and lifecycle invariants inside the transaction. + +#### Renew and expiration + +Lease expiration MUST use a backend-consistent time source. If multiple server instances share a backend, correctness MUST NOT depend on their wall clocks being perfectly synchronized. + +Expiration may be lazy during `lease` selection, eager via a sweeper, or both. A sweeper is an optimization and observability mechanism, not the sole correctness path. + +### Queue acceleration + +The current in-memory block channel can be retained only as a Data Engine implementation detail. A correct adapter follows these rules: + +1. The cache contains references/IDs or immutable snapshots, never exclusive ownership facts. +2. An item removed from the cache is not leased until the durable atomic lease succeeds. +3. Duplicate cache entries are tolerated and deduplicated by the durable transition. +4. Cache loss only affects performance. +5. Cache refill policy is invisible to the Server and COM engines. +6. Requested COM block size does not reconfigure global Data Engine queue structure. + +This permits a simple database-first reference adapter and a later high-throughput indexed adapter behind the same tests. + +### Pagination + +Page-number pagination SHOULD be replaced with opaque cursor pagination: + +```text +query(filter, sort, limit, after_cursor) -> items + next_cursor +``` + +- cursors are adapter-issued and opaque to clients; +- a cursor includes or authenticates the filter/sort context; +- maximum page size is enforced by Server policy and COM byte limits; +- consistency semantics are documented as snapshot or forward scan; +- cursor decoding errors map to `InvalidArgument`, not internal storage errors. + +The legacy gRPC adapter may emulate page numbers temporarily, with known performance limitations and a configured maximum reachable page. + +### Backend contract suite + +`engine-data-api` MUST ship a reusable conformance suite that every backend runs. It includes: + +- concurrent lease attempts never both receive a current lease for the same task; +- stale lease completion fails after re-lease; +- duplicate submit returns identical IDs; +- conflicting idempotency-key reuse fails; +- duplicate completion is idempotent; +- completion with changed output conflicts; +- expiry makes work available without depending on a sweeper tick; +- cancellation/deletion races obey the state machine; +- process-reopen tests preserve state and idempotency for durable adapters; +- cursor traversal has no duplicates or omissions under its documented consistency model; +- batch atomic/best-effort modes match their declaration; +- injected failures at commit boundaries do not expose half transitions. + +The in-memory adapter is the readable semantic reference. It is not automatically the production performance reference. + +## COM Engine + +COM is an adapter layer, not a second application server. + +### Responsibilities + +Every COM adapter MUST: + +- listen/connect using its technology; +- decode and structurally validate frames/messages; +- enforce message, stream, concurrency, and connection limits before allocating unbounded memory; +- extract credentials, peer information, deadlines, cancellation, request IDs, and trace context; +- map protocol DTOs to domain commands; +- call `ServerPort` only; +- map domain results and errors to protocol responses; +- enforce protocol-native backpressure; +- stop accepting new work during drain and allow bounded in-flight completion; +- emit adapter-level metrics without duplicating business metrics. + +It MUST NOT: + +- access a Data Engine or database; +- call task validators directly; +- decide authorization; +- create or remove leases; +- fire business lifecycle events; +- silently reinterpret partial failures; +- put tonic/protobuf/HTTP/broker types into Server commands. + +### gRPC adapter + +`engine-com-grpc` replaces the application logic currently in `engine/src/lib.rs` with mapping code. + +Legacy method mapping may be: + +| Existing RPC | Server Engine operation | Compatibility note | +|---|---|---| +| `CreateTaskBlock` | `submit(return_items=true)` | Preserve v1 response shape. | +| `EnqueueTaskBlock` | `submit(return_items=false)` | Return only after durable acceptance. | +| `AquireTaskBlock` | `lease` | Preserve misspelling in v1 only. | +| `AquireTaskStream` | `lease`, then frame items | Streaming is representation, not a separate lifecycle operation. | +| `PublishTaskBlock` | `complete` | v1 lacks lease tokens and needs a constrained compatibility policy. | +| `PublishTaskStream` | bounded stream-to-batch `complete` calls | Must report terminal failure consistently. | +| `AquireTaskReg` | `list_task_types` | Preserve v1 shape. | +| `GetTasks` | `query` | Page-to-cursor emulation is temporary. | +| `DeleteTaskBlock` | `administer` | Explicit state/precondition mapping. | +| `CheckAuth` | an auth/admin probe | Must not receive a DB handle. | +| `GetMetadata` | `metadata` | Adapter adds protocol version only if appropriate. | +| `cgrpc` | `invoke_extension` | Rename and redesign in v2. | + +### Error mapping for gRPC + +The adapter owns a documented mapping, for example: + +| Engine error | gRPC status | +|---|---| +| `InvalidArgument` | `INVALID_ARGUMENT` | +| `Unauthenticated` | `UNAUTHENTICATED` | +| `PermissionDenied` | `PERMISSION_DENIED` | +| `NotFound` | `NOT_FOUND` | +| `AlreadyExists` | `ALREADY_EXISTS` | +| `Conflict` / stale lease | `ABORTED` or `FAILED_PRECONDITION`, selected consistently by detail code | +| `ResourceExhausted` | `RESOURCE_EXHAUSTED` | +| `Cancelled` | `CANCELLED` | +| `DeadlineExceeded` | `DEADLINE_EXCEEDED` | +| transient Data failure | `UNAVAILABLE` | +| unexpected fault | `INTERNAL` with safe error ID | + +Mappings are tested as adapter conformance tests. Server code never constructs `tonic::Status`. + +### Deadlines and cancellation + +The gRPC adapter MUST extract client deadlines and cancellation. The Server Engine and Data Port receive the remaining budget/cancellation signal. Work spawned for a request must not become detached merely because the client disconnected. + +For state-changing operations, a deadline response is inherently ambiguous if commit raced with timeout. Idempotency keys are therefore required for safe retry. The adapter documentation MUST say that `DEADLINE_EXCEEDED` does not prove a mutation did not commit. + +### Streaming and backpressure + +- Incoming streams are read into bounded chunks; never `collect()` an unbounded stream. +- Chunk size is a COM configuration constrained by Server maximum batch size and Data backend limits. +- Outgoing streams use bounded channels and stop producing when cancellation fires. +- If a protocol supports acknowledgement per item, the adapter maps `BatchResult` without losing detail. +- If a legacy protocol cannot represent partial results, its compatibility mode MUST choose atomic behavior or fail the whole operation; it must not lie with a blanket success. +- Compression ratios and decompressed byte limits are enforced to prevent small frames expanding without bound. + +### Adding another communication mechanism + +A new COM adapter is complete when it: + +1. depends on `engine-server-api` and `engine-domain`, not Server implementation details; +2. maps every supported operation and error explicitly; +3. passes shared COM semantic tests; +4. has bounded load/backpressure behavior; +5. propagates request context; +6. has protocol-specific integration tests; +7. can be selected in `engine-runtime` configuration without recompiling Server or Data code, if dynamic selection is a project requirement. + +An HTTP adapter, for example, should contain routes and JSON DTOs only; it should invoke the same `ServerPort` instance as gRPC in a dual-listener process. + +## Authentication and authorization + +Auth is event-driven today and can receive a cloned sled handle. That contract must end. + +### Proposed ports + +```rust +pub trait Authenticator: Send + Sync { + async fn authenticate(&self, credential: &Credential, peer: Option<&PeerIdentity>) + -> Result; +} + +pub trait Authorizer: Send + Sync { + async fn authorize(&self, principal: &Principal, action: Action, resource: Resource) + -> Result; +} +``` + +- COM extracts but does not interpret protocol-carried credentials beyond safe syntax limits. +- Server authenticates and authorizes each operation. +- Data trusts the Server Engine call boundary in-process. If Data later becomes remote, service identity and authorization need a separate threat model. +- A missing credential is `Unauthenticated`; an identified principal lacking access is `PermissionDenied`. +- Default-allow authentication is forbidden in production profiles. Development bypasses must be explicit, loud in logs/health, and impossible to enable accidentally through a missing token. +- Admin and user actions use the same policy vocabulary, with distinct permissions. + +## Mods and events + +The current mod ABI conflicts with the new boundaries because `run(api: &mut ServerAPI)` exposes every subsystem and ties modules to Rust layout and compiler compatibility. + +### Required architectural change + +Mods MUST receive capability-limited interfaces, not `ServerAPI`, `sled::Db`, queue maps, or a COM service. + +Suggested capabilities: + +- register a task validator/descriptor; +- subscribe to specified lifecycle events; +- make bounded namespaced key/value reads and writes through a mod storage capability; +- register an extension command handler; +- emit structured telemetry; +- read a documented subset of immutable configuration. + +Capabilities are granted from metadata/config and default to least privilege. + +### Event phases + +Events are classified: + +- **Before-policy events**: synchronous, cancellable, run before the Data operation, may validate or transform explicitly mutable fields. +- **Committed domain events**: immutable facts returned/emitted only after the Data transition commits. +- **Delivery events**: asynchronous observations delivered through an outbox if loss across crashes is unacceptable. + +Post-success events MUST NOT be fired before durable commit. A failed observer must not roll back an already committed task unless the use case explicitly implements a compensating action. + +### ABI strategy + +The rewrite SHOULD replace `extern "Rust"` mods with a versioned interface definition. WebAssembly components with WIT are a strong candidate because WIT describes typed component contracts independently of Rust struct layout. This is a major compatibility decision and deserves its own RFC before implementation. + +During migration, a `LegacyModHost` MAY adapt a restricted subset of old events, but it MUST NOT recreate broad mutable access to the new runtime. Mods that depend on raw `ServerAPI` fields will require a major-version port. + +## Runtime and deployment + +### Composition root + +`engine-runtime` performs only construction and lifecycle management: + +```text +load and validate config + -> construct telemetry + -> open Data adapter and run schema checks/migrations + -> load task catalog and mod capabilities + -> construct Authenticator/Authorizer + -> construct ServerEngine + -> construct selected COM adapters with Arc + -> start readiness + -> serve until shutdown + -> stop readiness / stop accepting + -> drain with deadline + -> flush Data/telemetry and exit +``` + +No binary should mutate a partially initialized public API object. + +### Recommended initial topology + +```text +one OS process + engine-runtime + grpc adapter --------+ + optional http -------+--> one ServerEngine --> one DataPort implementation +``` + +Multiple COM adapters MAY listen simultaneously and share one Server Engine. This directly proves transport independence. + +### Optional future split + +COM may move to an edge process when independent scaling, network isolation, language diversity, or public-edge security justifies it. Doing so requires a new, private, versioned adapter between the remote COM process and `ServerPort`. The public client gRPC schema SHOULD NOT automatically become this internal protocol. + +Data may become remote only when a real Data service—not merely a database connection pool—is warranted. Most database-backed implementations should initially remain libraries that connect directly to the database. Avoid a pass-through “Data microservice” that adds latency but owns no meaningful semantics. + +### Horizontal scale + +The architecture permits multiple stateless Server/COM instances only if the selected Data adapter provides shared atomic lease semantics and the task catalog/policies are compatible across instances. + +Rolling deployments require: + +- version-compatible domain/data schemas; +- stable task validator versions or an explicit task schema version stored with each task; +- no process-local lease truth; +- backward-compatible COM API changes; +- a readiness gate until mods, schemas, and task catalog are loaded. + +## Configuration + +Configuration should be typed by owner: + +```text +[runtime] shutdown_grace, environment +[com.grpc] listen, max_frame_bytes, stream_chunk, concurrency, tls +[server] max_batch, lease_min, lease_default, lease_max, auth_policy +[data] adapter, durability, pool/cache/backend options +[mods] directories, capabilities, compatibility_mode +[telemetry] exporter, sampling, service_name +``` + +- Each engine receives only its validated config type. +- Environment overrides are parsed once in the runtime. +- Invalid production configuration fails startup with actionable errors; no `unwrap()`. +- Secrets use a secret provider/type and redact on `Debug`. +- Backend tuning such as sled cache bytes does not appear in Server or COM configuration objects. + +## Observability + +The boundary is only operable if one request can be followed across adapters and data operations. + +### Tracing + +- COM extracts or creates trace/request context. +- Server creates spans named by use case, not RPC method. +- Data creates backend-operation spans without recording payloads or credentials. +- Context is propagated across async tasks and future process boundaries. +- Untrusted incoming trace/baggage values are size-limited and sanitized; baggage must not carry secrets. + +Minimum span attributes include operation, task type where safe, batch count, result code, adapter name, and backend name. Task payload and auth material are forbidden. + +### Metrics + +At minimum: + +```text +engine_com_requests_total{adapter,operation,result} +engine_com_request_duration_seconds{adapter,operation} +engine_com_inflight{adapter,operation} +engine_server_operations_total{operation,result} +engine_server_operation_duration_seconds{operation} +engine_data_operations_total{backend,operation,result} +engine_data_operation_duration_seconds{backend,operation} +engine_tasks_submitted_total{task_type} +engine_tasks_leased_total{task_type} +engine_tasks_completed_total{task_type} +engine_lease_expired_total{task_type} +engine_task_queue_age_seconds{task_type} +engine_task_attempts{task_type} +engine_mod_hook_duration_seconds{mod,hook} +``` + +IDs such as task, user, lease, and request MUST NOT be metric labels because of cardinality. + +### Health + +- **Liveness**: process/event loop is functioning. It must not fail merely because a dependency is transiently down. +- **Readiness**: selected COM listener can serve and required Server/Data/catalog dependencies are ready. +- **Degraded status**: optional detail for mod failures, sweeper lag, or telemetry export without making every condition an outage. + +## API versioning + +External API versioning is independent from Rust crate versions, mod ABI versions, and Data schema versions. + +### Legacy gRPC v1 + +The current proto remains supported by a compatibility adapter during migration. It preserves wire field numbers and existing RPC names, including `Aquire`, while fixing only behavior that is explicitly declared a bug/security issue. + +### New API v2 + +A v2 API SHOULD: + +- use `Acquire`, `Submit`, and `Complete` terminology; +- include lease ID/revision in acquired and completed messages; +- include explicit idempotency keys for mutations; +- represent per-item batch results; +- distinguish task input and output; +- use cursor pagination; +- expose lease renewal/release if workers need them; +- define maximums and delivery semantics; +- reserve removed protobuf field numbers and names; +- expose capability/API versions in metadata. + +Do not make the domain structs serializable “as the protocol” and assume that is versioning. Each adapter owns stable schemas and conversion tests. + +## Migration plan + +This is a strangler migration around the current service, not a single unreviewable replacement commit. + +### Phase 0 — Ratify semantics and characterize behavior + +Before moving code: + +1. Approve the task state machine, at-least-once guarantee, stale-lease behavior, batch modes, auth defaults, and idempotency scope. +2. Add black-box characterization tests for every existing RPC, including malformed IDs, invalid payloads, mixed task types, partial batches, auth failure, duplicate publish, restart, stream cancellation, and backend failure. +3. Record benchmark baselines for submit, lease, complete, and end-to-end gRPC at representative payload/block/concurrency sizes. +4. Separate benchmark data directories and ensure benchmarks actually exercise concurrency and durable behavior. Simulated counter-only concurrency is not an architectural baseline. +5. Inventory which mods access which `ServerAPI` fields. + +Exit criterion: maintainers can state which legacy behaviors are compatibility requirements and which are bugs to remove. + +### Phase 1 — Introduce domain types and errors + +1. Add `engine-domain` with IDs, inputs/outputs, state, batch results, and error codes. +2. Add lossless conversion between current protobuf/task types and domain types in the legacy adapter area. +3. Keep existing handlers operational. +4. Add mapping and property tests; no persistence change yet. + +Exit criterion: no new application code accepts protobuf types outside COM conversion code. + +### Phase 2 — Extract the Data Engine behind current gRPC + +1. Add `engine-data-api` and the backend contract suite. +2. Implement `engine-data-memory` first to make semantics readable. +3. Implement `engine-data-sled` over current data, initially with compatibility behavior. +4. Move key construction, scans, batches, lease maps, refill cursors, active IDs, and cleanup out of `ServerAPI`/gRPC handlers. +5. Persist leases and idempotency, introducing a versioned schema. +6. Route current handlers through `DataPort` while responses remain unchanged. + +Exit criterion: gRPC code cannot access sled, queues, lease maps, or backend errors. + +### Phase 3 — Build the Server Engine + +1. Add `engine-server-api` and `engine-server`. +2. Move authn/authz, validation, task catalog, policy hooks, and lifecycle orchestration into use cases. +3. Return structured domain results/errors. +4. Make clocks, ID generation, auth, Data, catalog, and event publication injected dependencies. +5. Run all Server tests against `engine-data-memory` without tonic or sockets. + +Exit criterion: submit/lease/complete/query/admin behavior is fully testable by calling `ServerPort`. + +### Phase 4 — Reduce gRPC to a COM adapter + +1. Create `engine-com-grpc`. +2. Move generated proto code and tonic status mapping there. +3. Implement bounded streaming, deadline/cancellation propagation, limits, and graceful shutdown. +4. Run legacy black-box tests unchanged where behavior is intentionally compatible. +5. Add an in-process COM adapter or CLI test harness to prove transport neutrality. + +Exit criterion: changing Server/Data implementation does not rebuild protocol-neutral consumers, and a non-gRPC adapter completes the full task lifecycle. + +### Phase 5 — Data schema migration + +Choose one of these explicitly: + +- **Offline migration:** stop writes, back up, transform old keys to the new task/lease/idempotency schema, verify counts/digests, then start the new runtime. Recommended if downtime is acceptable. +- **Versioned online migration:** deploy code that can read old and new formats, write the new format, backfill with checkpoints, verify, then remove old reads in a later release. + +Avoid uncontrolled dual writes. A crash between independent old/new writes creates divergence unless both are covered by one backend transaction. Shadow reads and result comparison are safe; shadow mutation requires a designed replication/transaction mechanism. + +Required migration artifacts: + +- backup and restore procedure; +- version marker and resumable checkpoints; +- pre/post record counts by task state/type; +- payload digest verification; +- explicit mapping for current RAM leases (normally drain or expire before cutover); +- rollback rules that state when rollback becomes impossible after new writes. + +### Phase 6 — Mod API migration + +1. Freeze new uses of raw `ServerAPI` in mods. +2. Define capabilities and versioned task/event DTOs in a separate RFC. +3. Implement the new host and compatibility report tooling. +4. Port first-party mods. +5. Disable legacy native mods by default in the new major version. + +Exit criterion: no production mod can access a Data implementation or COM internals. + +### Phase 7 — API v2 and additional COM adapters + +1. Add the corrected v2 schema with lease/idempotency semantics. +2. Implement at least one genuinely different adapter or the embedded adapter as a conformance proof. +3. Document client upgrade and v1 deprecation windows. +4. Compare adapter behavior through a shared scenario suite. + +### Phase 8 — Remove legacy runtime + +Delete only after: + +- migrated data is verified; +- first-party clients/mods are ported; +- legacy adapter usage is below the agreed threshold; +- rollback window has closed; +- performance and fault tests meet gates. + +Targets for removal include public mutable `ServerAPI`, direct sled access outside Data, tonic types outside COM, and raw Rust-ABI runtime access from mods. + +## Mapping current files to the target + +| Current location | Target | +|---|---| +| `engine/src/bin/server.rs` | `engine-runtime` composition root | +| `engine/src/lib.rs` tonic service | mapping/framing in `engine-com-grpc`; use cases in `engine-server` | +| `engine/proto/engine.proto` | legacy schema owned by `engine-com-grpc`; v2 schema alongside it | +| `enginelib/src/api.rs` sled methods | `engine-data-sled` | +| `enginelib/src/api.rs` queue/refill/lease state | Data adapter internals or removed in database-first implementation | +| `enginelib/src/api.rs` registry/event/config fields | Server catalog/event host/typed configuration owners | +| `enginelib/src/task.rs` stored task/lease types | domain types; execution-specific `Task` trait moves to mod/worker SDK | +| `enginelib/src/event.rs` and `events/*` | capability-limited Server event host and immutable post-commit events | +| `enginelib/src/plugin.rs` | versioned mod host, eventually without `extern "Rust" ServerAPI` | +| `engine/tests/task_streaming_tests.rs` | COM gRPC integration tests plus protocol-neutral Server scenario tests | +| `engine/src/bin/no_grpc.rs` | real Server/Data benchmark, not direct internal-map manipulation | + +## Testing strategy + +### Test pyramid + +1. **Domain tests:** state transition tables, ID parsing boundaries, error/detail rules. +2. **Server use-case tests:** fake clock, fake auth, fake catalog, memory Data; no network. +3. **Data contract tests:** identical suite against memory, sled compatibility, and every future backend. +4. **COM conformance tests:** shared scenarios invoked through adapter-specific clients. +5. **gRPC integration tests:** metadata, statuses, deadlines, cancellation, streaming, reflection, limits. +6. **Fault tests:** crash/reopen, delayed commits, lost response simulation, cancellation at every await, expired leases, poisoned/unavailable mods. +7. **Property/model tests:** random operation sequences compared with a small reference state machine. +8. **Performance tests:** per-engine and end-to-end throughput/latency/resource measurements. + +### Required race tests + +- N workers lease one task concurrently; exactly one current lease wins. +- Old and new lease holders complete concurrently; only the new/current lease can first complete. +- Completion commits while the COM deadline expires; retry with idempotency returns the committed result. +- Delete/cancel races with lease and complete follow a declared ordering. +- Two Server instances submit the same idempotency key concurrently. +- Process restart occurs after durable lease commit but before response. +- Stream disconnect occurs mid-batch in atomic and best-effort modes. + +### Performance gates + +Numeric gates must be ratified from Phase 0 rather than invented here. The release gate MUST include: + +- p50/p95/p99 latency and throughput for submit/lease/complete; +- end-to-end gRPC and protocol-neutral baselines; +- CPU, allocations, memory, disk growth, and write amplification; +- empty queue and deep backlog behavior; +- 1, 8, and high-concurrency worker cases; +- small and large payloads; +- cache-cold restart behavior; +- correctness checks enabled during the benchmark. + +No result may be extrapolated from a small test to one billion tasks as an acceptance substitute. + +## Security considerations + +- Remove permissive auth behavior from production defaults. +- Bound decoded message count, individual payload bytes, aggregate batch bytes, stream duration, and in-flight work per principal/connection. +- Treat task namespace/name, IDs, cursor, metadata, and mod extension inputs as untrusted. +- Do not expose backend errors or paths. +- Lease IDs and idempotency keys are sensitive capability-like values and must not be metric labels. +- Authorize queries and admin operations by task namespace/type, not merely by endpoint. +- Mods execute with explicit capabilities and resource budgets; a mod must not block the async runtime indefinitely. +- If COM is remote from Server, authenticate the COM service and do not trust caller-supplied principals without cryptographic/channel identity. +- If Data is remote, encrypt/authenticate the link and re-evaluate which invariant checks remain server-side versus data-side. +- Trace/baggage propagation is sanitized at trust boundaries. + +## Failure and shutdown semantics + +| Failure | Required behavior | +|---|---| +| COM listener failure | Other listeners MAY continue; readiness reflects configured required listeners. | +| Client disconnect | Cancel request-scoped work where safe; committed mutations remain committed. | +| Data transient failure | Return structured retryable `Unavailable`; do not report success. | +| Data ambiguous commit | Retry via idempotency; do not blindly replay with a new key. | +| Mod before-hook failure | Follow configured fail-open/fail-closed policy per hook; auth/security hooks default fail-closed. | +| Post-commit observer failure | Record/retry through outbox if durable delivery is configured; never pretend commit failed. | +| Lease sweeper failure | Lazy expiry keeps correctness; readiness/alerts expose lag if it risks capacity. | +| Panic in request task | Convert at outer boundary to internal error, record error ID, keep process alive where unwind safety permits. | +| Shutdown | Stop readiness/acceptance, cancel or drain bounded in-flight calls, close adapters, flush Data/telemetry. | + +## Alternatives considered + +### A. Put a trait around the current `EngineService` + +Rejected. It would hide but preserve tonic requests, shared `ServerAPI`, mixed persistence and policy, and non-atomic lease behavior. A mock of a coupled object is still coupled architecture. + +### B. Extract only COM and leave Server/Data together + +This would meet the narrow goal of adding HTTP, but storage and lifecycle correctness would remain inseparable. Given the rewrite scope, the Data boundary is worth defining now. Implementation may still land COM and Data extraction in phases. + +### C. Make COM, Server, and Data three microservices immediately + +Rejected for the initial topology. It adds operational and consistency cost without being necessary for replaceability. The logical boundaries preserve this future option. + +### D. Use a message broker as the whole Data Engine + +Not selected as the universal model. Brokers can provide excellent delivery primitives, but the engine also needs task metadata, state queries, admin operations, results, idempotency, and mod-defined task catalogs. A broker-backed Data adapter MAY be built if it satisfies the entire contract and documents its consistency limitations. + +### E. Adopt full event sourcing + +Deferred. An append-only event log could improve audit/rebuild behavior, but it makes projections, schema evolution, storage volume, and operational recovery central to the rewrite. The proposed committed events/outbox preserve an evolution path without requiring all state to be reconstructed from events today. + +### F. Keep public `ServerAPI` as a compatibility facade forever + +Rejected. It would become a permanent escape hatch around every boundary. A temporary facade may exist only with a removal milestone and no access to new internals. + +### G. Select PostgreSQL now and design directly around SQL + +Not required by this RFC. PostgreSQL is a strong candidate for shared, transactional multi-server leasing; an embedded backend may be preferable for simple deployments. The contract suite should make the choice evidence-based. SQL types and queries must remain inside the adapter either way. + +## Risks and mitigations + +| Risk | Mitigation | +|---|---| +| Rewrite stalls while features continue landing | Phase by endpoint/use case; freeze new direct dependencies; keep legacy adapter running. | +| Abstraction harms throughput | Batch-oriented ports, benchmark each boundary, static/generic composition where useful, optimize adapters behind contracts. | +| Data abstraction becomes lowest-common-denominator CRUD | Define semantic atomic operations and explicit backend capability checks. | +| Legacy gRPC cannot express new lease/idempotency semantics | Constrained v1 compatibility policy plus v2; do not weaken internal invariants indefinitely. | +| Mod ecosystem breaks | Inventory usage, capability RFC/tooling, compatibility report, major-version communication. | +| Dual storage migration corrupts data | Prefer offline migration or transactional/versioned online plan; digest verification and checkpoints. | +| Three “engines” become three god objects | Split ports by purposeful conversation internally; keep facade small; enforce dependencies. | +| Too many tiny crates slow development | Use crates only for enforced architectural boundaries; modules may organize internals. | +| In-memory optimization reintroduces split-brain truth | Durable state remains authoritative; cache duplication/loss covered by contract tests. | +| Error compatibility changes clients | Golden status/detail tests and versioned API mapping. | + +## Acceptance criteria + +The rewrite described by this RFC is complete when all of the following are true: + +1. The complete submit → lease → complete lifecycle runs through `ServerPort` with the memory Data adapter and no tonic/sled dependency. +2. The same Server Engine instance can be driven by gRPC and at least one non-gRPC/embedded COM adapter. +3. Swapping memory/sled/another Data adapter requires no Server or COM source changes beyond runtime composition/configuration. +4. All Data adapters pass the shared atomicity, lease, idempotency, expiry, pagination, and reopen contract tests applicable to their durability class. +5. No Server Engine source imports tonic, prost, sled, generated protobuf types, or backend keys. +6. No COM source accesses Data directly. +7. Leases are durable and fenced; stale completion is rejected. +8. Mutating operations have documented idempotency and ambiguous-timeout behavior. +9. Batches return explicit item outcomes or are explicitly atomic. +10. Production auth is not default-allow, and auth code has no raw database handle. +11. Mods use capabilities rather than mutable runtime/database access, or legacy mods are clearly isolated and disabled by default for the new major version. +12. Graceful shutdown, deadline, cancellation, resource-limit, and trace-context behavior is tested. +13. Data migration and rollback procedures have been exercised on a production-sized copy. +14. Workspace tests, builds, lint policy, fault tests, and ratified performance gates pass. +15. Architecture dependency rules run in CI. + +## Decisions required before implementation + +These are blocking architecture decisions, not details to leave to the first implementer: + +1. Is the first production Data backend a compatibility sled adapter, a new embedded backend, PostgreSQL, or two tiers? +2. What exact lease expiry rule applies when completion and expiry race? +3. What are default/min/max lease durations and maximum attempts? +4. Are submit and complete batches atomic or best-effort by default in v2? +5. What is the idempotency scope and retention period: principal + operation + key, tenant + key, or global key? +6. Does task output stay inline, move to blob storage above a threshold, or remain application-defined bytes? +7. What ordering/fairness promise does lease selection make? +8. Which legacy partial-success behaviors are bugs versus compatibility requirements? +9. How long is gRPC v1 supported, and how can its lack of lease tokens be safely constrained? +10. Is WASM/WIT adopted for mods, or is another stable interface chosen? +11. Which hooks require durable post-commit delivery through an outbox? +12. Which COM listeners are required for readiness when several run together? + +## Recommended first implementation slice + +Do not start by moving files. Start with one vertical slice: + +1. Add domain `TaskType`, `TaskId`, `LeaseId`, `TaskInput`, `TaskOutput`, batch outcomes, and errors. +2. Specify `submit`, `lease`, and `complete` only. +3. Implement their memory Data state machine and conformance tests. +4. Implement Server use cases with fake auth/catalog/clock. +5. Adapt only `CreateTaskBlock`, `AquireTaskBlock`, and `PublishTaskBlock` to that Server port behind a feature/config switch. +6. Add an embedded test adapter that calls the same port. +7. Run characterization, race, restart, and performance comparisons. + +This slice proves all three boundaries before the team migrates queries, streams, admin RPCs, extensions, and mods. + +## Research basis + +This design is informed by the following primary or official sources: + +- Alistair Cockburn's original [Hexagonal Architecture / Ports and Adapters](https://alistair.cockburn.us/hexagonal-architecture) describes technology-specific adapters around purposeful application ports and explicitly includes alternate databases and test adapters. +- The official gRPC guides document [deadlines](https://grpc.io/docs/guides/deadlines/), [cancellation](https://grpc.io/docs/guides/cancellation/), [status codes](https://grpc.io/docs/guides/status-codes/), and [graceful shutdown](https://grpc.io/docs/guides/server-graceful-stop/). These behaviors belong in COM context propagation and mapping, while the Server must cooperate with cancellation. +- The official [Protocol Buffers proto3 guide](https://protobuf.dev/programming-guides/proto3/) documents wire-safe evolution and the need to reserve removed field numbers/names. +- OpenTelemetry's official [context propagation](https://opentelemetry.io/docs/concepts/context-propagation/) and [propagator specification](https://opentelemetry.io/docs/specs/otel/context/api-propagators/) motivate a protocol-neutral request context and warn about trust-boundary/baggage handling. +- [RFC 9562](https://www.rfc-editor.org/rfc/rfc9562.html) standardizes UUIDv7 and its time-ordered layout. +- The Bytecode Alliance's [WIT reference](https://component-model.bytecodealliance.org/design/wit.html) explains WIT as a language for typed component contracts; it supports, but does not by itself decide, the proposed mod ABI direction. +- The Rust Reference's [dyn compatibility rules](https://doc.rust-lang.org/reference/items/traits.html#dyn-compatibility) are why the RFC leaves generic versus erased async port dispatch as an explicit implementation decision. + +## Relationship to previous RFCs + +- RFC 1002's `.rf` packaging may remain as a container during migration, but its native `run(&mut EngineAPI)` ABI conflicts with this RFC and requires a successor mod-interface RFC. +- RFC 1003 client hooks remain conceptually client-side; protocol DTO changes in v2 require updated client mappings. +- RFC 1004 server lifecycle events are preserved as use-case concepts, but their firing location and post-commit semantics move into the Server Engine. +- RFC 1005's performance goals become Data Engine implementation work constrained by the state-machine and contract tests in this RFC. +