diff --git a/.github/workflows/formal-checks.yml b/.github/workflows/formal-checks.yml index 87e256e..61796ae 100644 --- a/.github/workflows/formal-checks.yml +++ b/.github/workflows/formal-checks.yml @@ -76,27 +76,51 @@ jobs: # toolchain is identical), so the first shard to finish install # warms the cache for all the others on subsequent runs. kani-shard: - name: kani-shard / ${{ matrix.harness }} + name: kani-shard / ${{ matrix.crate }} :: ${{ matrix.harness }} runs-on: ubuntu-latest timeout-minutes: 15 strategy: - # `fail-fast: false` lets all 9 harnesses report results even + # `fail-fast: false` lets all harnesses report results even # if one regresses; otherwise GHA cancels the rest and we lose # signal on the others. fail-fast: false matrix: - # Fully-qualified harness paths from - # `crates/kcore-sanitize/src/lib.rs::kani_proofs`. - harness: - - kani_proofs::nix_escape_never_panics - - kani_proofs::nix_escape_output_is_always_safe - - kani_proofs::sanitize_nix_attr_key_preserves_char_count - - kani_proofs::sanitize_nix_attr_key_charset - - kani_proofs::dot_dot_check_never_panics - - kani_proofs::assert_safe_path_never_panics - - kani_proofs::assert_safe_path_acceptance_implies_safe - - kani_proofs::segment_validation_never_panics - - kani_proofs::segment_acceptance_implies_safe + # Fully-qualified harness paths from each crate's + # `kani_proofs` module. `crate` selects which workspace + # member `cargo kani -p` operates on. + include: + - crate: kcore-sanitize + harness: kani_proofs::nix_escape_never_panics + - crate: kcore-sanitize + harness: kani_proofs::nix_escape_output_is_always_safe + - crate: kcore-sanitize + harness: kani_proofs::sanitize_nix_attr_key_preserves_char_count + - crate: kcore-sanitize + harness: kani_proofs::sanitize_nix_attr_key_charset + - crate: kcore-sanitize + harness: kani_proofs::dot_dot_check_never_panics + - crate: kcore-sanitize + harness: kani_proofs::assert_safe_path_never_panics + - crate: kcore-sanitize + harness: kani_proofs::assert_safe_path_acceptance_implies_safe + - crate: kcore-sanitize + harness: kani_proofs::segment_validation_never_panics + - crate: kcore-sanitize + harness: kani_proofs::segment_acceptance_implies_safe + # `kcore-disko-types` is the shared, std-only crate used by + # both the controller pre-flight classifier and the node- + # agent authoritative classifier. The harnesses below cover + # the layout-diff parser (`extract_target_devices` + + # `strip_nix_comments`) — its post-conditions are what + # downstream classifier safety arguments rest on. + - crate: kcore-disko-types + harness: kani_proofs::extract_target_devices_never_panics + - crate: kcore-disko-types + harness: kani_proofs::strip_nix_comments_preserves_length + - crate: kcore-disko-types + harness: kani_proofs::extract_target_devices_outputs_dev_prefixed_paths + - crate: kcore-disko-types + harness: kani_proofs::extract_target_devices_is_deterministic # Pinned kani-verifier version. Bumping this single env var # (here AND in the cache key below) is the only change needed # to roll forward; the install step honours it and the cache @@ -125,7 +149,7 @@ jobs: - name: Cache cargo registry + target uses: Swatinem/rust-cache@v2 with: - shared-key: kani-${{ matrix.harness }} + shared-key: kani-${{ matrix.crate }}-${{ matrix.harness }} # Caches the Kani toolchain itself (CBMC + symex binaries + # ~/.kani layout). Cold install of kani-verifier from crates.io @@ -216,11 +240,12 @@ jobs: ~/.kani key: ${{ steps.kani-cache.outputs.cache-primary-key }} - # All proofs live in `kcore-sanitize`, a leaf crate with **no - # non-std dependencies**, so Kani only has to compile that one - # crate (a few seconds) instead of the full controller dep - # graph (rusqlite-bundled SQLite, rcgen+aws-lc-rs, tonic, ... - # which takes >20 min on a 4-vCPU GHA runner). + # Each proof lives in a leaf crate with **no non-std + # dependencies** (`kcore-sanitize`, `kcore-disko-types`), so + # Kani only has to compile that one crate (a few seconds) + # instead of the full controller dep graph (rusqlite-bundled + # SQLite, rcgen+aws-lc-rs, tonic, ... which takes >20 min on + # a 4-vCPU GHA runner). # # `--harness` is a substring match; we anchor with the full # `kani_proofs::` module path so a future harness with a @@ -232,7 +257,7 @@ jobs: run: | set -euo pipefail cargo-kani --version - cargo kani -p kcore-sanitize \ + cargo kani -p '${{ matrix.crate }}' \ --harness '${{ matrix.harness }}' \ --output-format terse diff --git a/Cargo.lock b/Cargo.lock index a7d2464..c4d8640 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1417,6 +1417,7 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", + "kcore-disko-types", "kcore-sanitize", "libc", "pem", @@ -1467,6 +1468,13 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "kcore-disko-types" +version = "0.1.0" +dependencies = [ + "proptest", +] + [[package]] name = "kcore-kctl" version = "0.1.0" @@ -1505,6 +1513,7 @@ dependencies = [ "hyper", "hyper-util", "hyperlocal", + "kcore-disko-types", "kcore-sanitize", "libc", "pem", diff --git a/Cargo.toml b/Cargo.toml index 83d7a4c..ca1f8db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "crates/kcore-sanitize", + "crates/kcore-disko-types", "crates/node-agent", "crates/controller", "crates/kctl", diff --git a/Makefile b/Makefile index 149a81f..7e12925 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ iso: @echo "" @ls -lh result-iso/iso/*.iso @echo "" - @echo "ISO built: result-iso/iso/nixos-kcore-$(VERSION)-x86_64-linux.iso" + @echo "ISO built: result-iso/iso/kcoreos-$(VERSION)-x86_64-linux.iso" iso-remote: @echo "Building kcore ISO $(V) on remote Linux server..." diff --git a/crates/controller/Cargo.toml b/crates/controller/Cargo.toml index 91670f8..174266c 100644 --- a/crates/controller/Cargo.toml +++ b/crates/controller/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [dependencies] anyhow = "1.0.102" kcore-sanitize = { path = "../kcore-sanitize" } +kcore-disko-types = { path = "../kcore-disko-types" } clap = { version = "4", features = ["derive"] } pem = "3" prost = "0.13" diff --git a/crates/controller/src/db.rs b/crates/controller/src/db.rs index cab0d64..c237f4a 100644 --- a/crates/controller/src/db.rs +++ b/crates/controller/src/db.rs @@ -128,6 +128,26 @@ pub struct SecurityGroupVmAttachmentRow { pub vm_id: String, } +#[derive(Debug, Clone)] +pub struct DiskLayoutRow { + pub name: String, + pub node_id: String, + pub generation: i64, + pub layout_nix: String, + pub created_at: String, + pub updated_at: String, +} + +#[derive(Debug, Clone)] +pub struct DiskLayoutStatusRow { + pub name: String, + pub observed_generation: i64, + pub phase: String, + pub refusal_reason: String, + pub message: String, + pub last_transition_at: String, +} + #[derive(Debug, Clone)] pub struct SecurityGroupNetworkAttachmentRow { pub security_group: String, @@ -362,6 +382,23 @@ impl Database { network_name TEXT NOT NULL, node_id TEXT NOT NULL REFERENCES nodes(id) ON DELETE CASCADE, PRIMARY KEY (security_group, network_name, node_id) + ); + CREATE TABLE IF NOT EXISTS disk_layouts ( + name TEXT PRIMARY KEY, + node_id TEXT NOT NULL REFERENCES nodes(id) ON DELETE CASCADE, + generation INTEGER NOT NULL DEFAULT 1, + layout_nix TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE INDEX IF NOT EXISTS idx_disk_layouts_node ON disk_layouts(node_id); + CREATE TABLE IF NOT EXISTS disk_layout_status ( + name TEXT PRIMARY KEY REFERENCES disk_layouts(name) ON DELETE CASCADE, + observed_generation INTEGER NOT NULL DEFAULT 0, + phase TEXT NOT NULL DEFAULT 'pending', + refusal_reason TEXT NOT NULL DEFAULT '', + message TEXT NOT NULL DEFAULT '', + last_transition_at TEXT NOT NULL DEFAULT (datetime('now')) );", )?; @@ -725,7 +762,29 @@ impl Database { ); } - const CURRENT_VERSION: i32 = 26; + if version < 27 { + let _ = conn.execute_batch( + "CREATE TABLE IF NOT EXISTS disk_layouts ( + name TEXT PRIMARY KEY, + node_id TEXT NOT NULL REFERENCES nodes(id) ON DELETE CASCADE, + generation INTEGER NOT NULL DEFAULT 1, + layout_nix TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE INDEX IF NOT EXISTS idx_disk_layouts_node ON disk_layouts(node_id); + CREATE TABLE IF NOT EXISTS disk_layout_status ( + name TEXT PRIMARY KEY REFERENCES disk_layouts(name) ON DELETE CASCADE, + observed_generation INTEGER NOT NULL DEFAULT 0, + phase TEXT NOT NULL DEFAULT 'pending', + refusal_reason TEXT NOT NULL DEFAULT '', + message TEXT NOT NULL DEFAULT '', + last_transition_at TEXT NOT NULL DEFAULT (datetime('now')) + );", + ); + } + + const CURRENT_VERSION: i32 = 27; if version < CURRENT_VERSION { conn.execute("DELETE FROM schema_version", [])?; conn.execute( @@ -1944,6 +2003,202 @@ impl Database { Ok(rows > 0) } + pub fn upsert_disk_layout( + &self, + layout: &DiskLayoutRow, + ) -> Result { + let conn = self.lock_conn()?; + conn.execute( + "INSERT INTO disk_layouts (name, node_id, generation, layout_nix) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(name) DO UPDATE SET + node_id = excluded.node_id, + generation = excluded.generation, + layout_nix = excluded.layout_nix, + updated_at = datetime('now')", + params![ + layout.name, + layout.node_id, + layout.generation, + layout.layout_nix, + ], + )?; + let mut stmt = conn.prepare( + "SELECT name, node_id, generation, layout_nix, created_at, updated_at + FROM disk_layouts WHERE name = ?1", + )?; + let row = stmt.query_row(params![layout.name], |row| { + Ok(DiskLayoutRow { + name: row.get(0)?, + node_id: row.get(1)?, + generation: row.get(2)?, + layout_nix: row.get(3)?, + created_at: row.get(4)?, + updated_at: row.get(5)?, + }) + })?; + Ok(row) + } + + pub fn get_disk_layout(&self, name: &str) -> Result, rusqlite::Error> { + let conn = self.lock_conn()?; + let mut stmt = conn.prepare( + "SELECT name, node_id, generation, layout_nix, created_at, updated_at + FROM disk_layouts WHERE name = ?1", + )?; + let mut rows = stmt.query_map(params![name], |row| { + Ok(DiskLayoutRow { + name: row.get(0)?, + node_id: row.get(1)?, + generation: row.get(2)?, + layout_nix: row.get(3)?, + created_at: row.get(4)?, + updated_at: row.get(5)?, + }) + })?; + rows.next().transpose() + } + + pub fn list_disk_layouts( + &self, + node_id_filter: Option<&str>, + ) -> Result, rusqlite::Error> { + let conn = self.lock_conn()?; + if let Some(node_id) = node_id_filter { + let mut stmt = conn.prepare( + "SELECT name, node_id, generation, layout_nix, created_at, updated_at + FROM disk_layouts WHERE node_id = ?1 ORDER BY name ASC", + )?; + let rows = stmt.query_map(params![node_id], |row| { + Ok(DiskLayoutRow { + name: row.get(0)?, + node_id: row.get(1)?, + generation: row.get(2)?, + layout_nix: row.get(3)?, + created_at: row.get(4)?, + updated_at: row.get(5)?, + }) + })?; + rows.collect() + } else { + let mut stmt = conn.prepare( + "SELECT name, node_id, generation, layout_nix, created_at, updated_at + FROM disk_layouts ORDER BY name ASC", + )?; + let rows = stmt.query_map([], |row| { + Ok(DiskLayoutRow { + name: row.get(0)?, + node_id: row.get(1)?, + generation: row.get(2)?, + layout_nix: row.get(3)?, + created_at: row.get(4)?, + updated_at: row.get(5)?, + }) + })?; + rows.collect() + } + } + + pub fn delete_disk_layout(&self, name: &str) -> Result { + let conn = self.lock_conn()?; + let rows = conn.execute("DELETE FROM disk_layouts WHERE name = ?1", params![name])?; + Ok(rows > 0) + } + + pub fn upsert_disk_layout_status( + &self, + status: &DiskLayoutStatusRow, + ) -> Result<(), rusqlite::Error> { + let conn = self.lock_conn()?; + conn.execute( + "INSERT INTO disk_layout_status ( + name, observed_generation, phase, refusal_reason, message, last_transition_at + ) VALUES (?1, ?2, ?3, ?4, ?5, datetime('now')) + ON CONFLICT(name) DO UPDATE SET + observed_generation = excluded.observed_generation, + phase = excluded.phase, + refusal_reason = excluded.refusal_reason, + message = excluded.message, + last_transition_at = datetime('now')", + params![ + status.name, + status.observed_generation, + status.phase, + status.refusal_reason, + status.message, + ], + )?; + Ok(()) + } + + pub fn get_disk_layout_status( + &self, + name: &str, + ) -> Result, rusqlite::Error> { + let conn = self.lock_conn()?; + let mut stmt = conn.prepare( + "SELECT name, observed_generation, phase, refusal_reason, message, last_transition_at + FROM disk_layout_status WHERE name = ?1", + )?; + let mut rows = stmt.query_map(params![name], |row| { + Ok(DiskLayoutStatusRow { + name: row.get(0)?, + observed_generation: row.get(1)?, + phase: row.get(2)?, + refusal_reason: row.get(3)?, + message: row.get(4)?, + last_transition_at: row.get(5)?, + }) + })?; + rows.next().transpose() + } + + pub fn list_disk_layout_statuses(&self) -> Result, rusqlite::Error> { + let conn = self.lock_conn()?; + let mut stmt = conn.prepare( + "SELECT name, observed_generation, phase, refusal_reason, message, last_transition_at + FROM disk_layout_status", + )?; + let rows = stmt.query_map([], |row| { + Ok(DiskLayoutStatusRow { + name: row.get(0)?, + observed_generation: row.get(1)?, + phase: row.get(2)?, + refusal_reason: row.get(3)?, + message: row.get(4)?, + last_transition_at: row.get(5)?, + }) + })?; + rows.collect() + } + + /// Return disk layouts whose status has `observed_generation < generation` + /// (or no status row yet). These are the rows the reconciler should push + /// to the node-agent on the next tick. + pub fn list_disk_layouts_needing_reconcile( + &self, + ) -> Result, rusqlite::Error> { + let conn = self.lock_conn()?; + let mut stmt = conn.prepare( + "SELECT d.name, d.node_id, d.generation, d.layout_nix, d.created_at, d.updated_at + FROM disk_layouts d + LEFT JOIN disk_layout_status s ON s.name = d.name + WHERE s.name IS NULL OR s.observed_generation < d.generation + ORDER BY d.name ASC", + )?; + let rows = stmt.query_map([], |row| { + Ok(DiskLayoutRow { + name: row.get(0)?, + node_id: row.get(1)?, + generation: row.get(2)?, + layout_nix: row.get(3)?, + created_at: row.get(4)?, + updated_at: row.get(5)?, + }) + })?; + rows.collect() + } + pub fn replace_security_group_rules( &self, security_group: &str, @@ -3863,5 +4118,206 @@ mod proptests { prop_assert_eq!(&after_first.status, &after_second.status); prop_assert_eq!(&after_first.approval_status, &after_second.approval_status); } + + /// **DiskLayout CRUD round-trip**: for any randomised + /// `DiskLayoutRow` whose `node_id` matches an existing node, + /// `upsert_disk_layout` followed by `get_disk_layout` returns + /// a row whose user-set fields equal the inserted ones. + #[test] + fn disk_layout_upsert_then_get_returns_same_row( + node_id in "[a-z0-9-]{1,8}", + name in "[a-z0-9-]{1,12}", + generation in 1i64..=1024, + layout_nix in "[ -~\n]{0,256}", + ) { + let db = Database::open(":memory:").expect("open db"); + db.upsert_node(&make_node( + &node_id, "h", "127.0.0.1:9091", + 4, 1024, 0, 0, false, 0, "DC1", + )).unwrap(); + + let layout = DiskLayoutRow { + name: name.clone(), + node_id: node_id.clone(), + generation, + layout_nix: layout_nix.clone(), + created_at: String::new(), + updated_at: String::new(), + }; + db.upsert_disk_layout(&layout).unwrap(); + + let got = db.get_disk_layout(&name).unwrap().expect("layout exists"); + prop_assert_eq!(&got.name, &name); + prop_assert_eq!(&got.node_id, &node_id); + prop_assert_eq!(got.generation, generation); + prop_assert_eq!(&got.layout_nix, &layout_nix); + prop_assert!(!got.created_at.is_empty()); + prop_assert!(!got.updated_at.is_empty()); + } + + /// **DiskLayout upsert is idempotent**: applying the same row + /// twice yields exactly one row in `list_disk_layouts`. + #[test] + fn disk_layout_upsert_is_idempotent( + node_id in "[a-z0-9-]{1,8}", + name in "[a-z0-9-]{1,12}", + ) { + let db = Database::open(":memory:").expect("open db"); + db.upsert_node(&make_node( + &node_id, "h", "127.0.0.1:9091", + 4, 1024, 0, 0, false, 0, "DC1", + )).unwrap(); + + let layout = DiskLayoutRow { + name: name.clone(), + node_id: node_id.clone(), + generation: 1, + layout_nix: "{ disko.devices = {}; }".to_string(), + created_at: String::new(), + updated_at: String::new(), + }; + db.upsert_disk_layout(&layout).unwrap(); + db.upsert_disk_layout(&layout).unwrap(); + + let listed = db.list_disk_layouts(None).unwrap(); + prop_assert_eq!(listed.len(), 1); + prop_assert_eq!(&listed[0].name, &name); + + let only_node = db.list_disk_layouts(Some(&node_id)).unwrap(); + prop_assert_eq!(only_node.len(), 1); + } + + /// **DiskLayout FK integrity**: inserting a layout whose + /// `node_id` does not exist must fail with a SQLite FK + /// constraint violation. + #[test] + fn disk_layout_rejects_unknown_node_id( + present_node in "[a-z0-9-]{1,8}", + missing_node in "[a-z0-9-]{1,8}", + name in "[a-z0-9-]{1,12}", + ) { + prop_assume!(present_node != missing_node); + let db = Database::open(":memory:").expect("open db"); + db.upsert_node(&make_node( + &present_node, "h", "127.0.0.1:9091", + 1, 1024, 0, 0, false, 0, "DC1", + )).unwrap(); + + let layout = DiskLayoutRow { + name, + node_id: missing_node, + generation: 1, + layout_nix: "{}".to_string(), + created_at: String::new(), + updated_at: String::new(), + }; + const SQLITE_CONSTRAINT_FOREIGNKEY: i32 = 787; + match db.upsert_disk_layout(&layout) { + Ok(_) => prop_assert!(false, "expected FK violation, got Ok"), + Err(rusqlite::Error::SqliteFailure(sqlite_err, _)) + if sqlite_err.code == rusqlite::ErrorCode::ConstraintViolation + && sqlite_err.extended_code == SQLITE_CONSTRAINT_FOREIGNKEY => {} + Err(other) => prop_assert!( + false, + "expected SQLITE_CONSTRAINT_FOREIGNKEY, got: {other}" + ), + } + } + + /// **DiskLayout delete removes from every view**: after + /// `delete_disk_layout`, the layout disappears from + /// `get_disk_layout` and `list_disk_layouts` and any status + /// row is cascaded away. + #[test] + fn disk_layout_delete_removes_from_all_views( + node_id in "[a-z0-9-]{1,8}", + name in "[a-z0-9-]{1,12}", + ) { + let db = Database::open(":memory:").expect("open db"); + db.upsert_node(&make_node( + &node_id, "h", "127.0.0.1:9091", + 4, 1024, 0, 0, false, 0, "DC1", + )).unwrap(); + + let layout = DiskLayoutRow { + name: name.clone(), + node_id: node_id.clone(), + generation: 7, + layout_nix: "{}".to_string(), + created_at: String::new(), + updated_at: String::new(), + }; + db.upsert_disk_layout(&layout).unwrap(); + db.upsert_disk_layout_status(&DiskLayoutStatusRow { + name: name.clone(), + observed_generation: 7, + phase: "applied".to_string(), + refusal_reason: String::new(), + message: "ok".to_string(), + last_transition_at: String::new(), + }).unwrap(); + + let deleted = db.delete_disk_layout(&name).unwrap(); + prop_assert!(deleted); + prop_assert!(db.get_disk_layout(&name).unwrap().is_none()); + prop_assert!(db.get_disk_layout_status(&name).unwrap().is_none()); + let listed = db.list_disk_layouts(None).unwrap(); + prop_assert!(!listed.iter().any(|l| l.name == name)); + } + + /// **Reconciler queue invariant**: a layout shows up in + /// `list_disk_layouts_needing_reconcile` iff it has no status + /// row, or its status `observed_generation` is strictly + /// behind the layout's `generation`. Once status catches up, + /// it must drop out of the queue. + #[test] + fn disk_layout_reconcile_queue_tracks_observed_generation( + node_id in "[a-z0-9-]{1,8}", + name in "[a-z0-9-]{1,12}", + generation in 1i64..=64, + ) { + let db = Database::open(":memory:").expect("open db"); + db.upsert_node(&make_node( + &node_id, "h", "127.0.0.1:9091", + 4, 1024, 0, 0, false, 0, "DC1", + )).unwrap(); + + db.upsert_disk_layout(&DiskLayoutRow { + name: name.clone(), + node_id: node_id.clone(), + generation, + layout_nix: "{}".to_string(), + created_at: String::new(), + updated_at: String::new(), + }).unwrap(); + + let pending = db.list_disk_layouts_needing_reconcile().unwrap(); + prop_assert!(pending.iter().any(|l| l.name == name), + "freshly created layout must be queued for reconcile"); + + db.upsert_disk_layout_status(&DiskLayoutStatusRow { + name: name.clone(), + observed_generation: generation, + phase: "applied".to_string(), + refusal_reason: String::new(), + message: String::new(), + last_transition_at: String::new(), + }).unwrap(); + let after = db.list_disk_layouts_needing_reconcile().unwrap(); + prop_assert!(!after.iter().any(|l| l.name == name), + "layout with observed_generation == generation must NOT be queued"); + + db.upsert_disk_layout(&DiskLayoutRow { + name: name.clone(), + node_id: node_id.clone(), + generation: generation + 1, + layout_nix: "{ updated = true; }".to_string(), + created_at: String::new(), + updated_at: String::new(), + }).unwrap(); + let bumped = db.list_disk_layouts_needing_reconcile().unwrap(); + prop_assert!(bumped.iter().any(|l| l.name == name), + "layout whose generation moved past observed_generation must be re-queued"); + } } } diff --git a/crates/controller/src/disk_reconciler.rs b/crates/controller/src/disk_reconciler.rs new file mode 100644 index 0000000..61a9db4 --- /dev/null +++ b/crates/controller/src/disk_reconciler.rs @@ -0,0 +1,148 @@ +//! DiskLayout reconciler. +//! +//! Every tick, list every `disk_layouts` row whose most recent +//! `disk_layout_status.observed_generation` is behind the row's `generation` +//! (or has no status yet), connect to the owning node, and call +//! [`ApplyDiskLayout`] with `apply = true` and `rebuild = true`. +//! +//! The reconciler never touches VMs. If the node-agent refuses the change +//! (classifier says the disk is not idle), the refusal code is recorded in +//! `disk_layout_status` and the operator is expected to drain VMs off the +//! affected disks and resubmit the same manifest — the generation does not +//! bump on unchanged content, so the reconciler will retry the exact same +//! payload until the node accepts or the operator deletes the layout. + +use std::time::Duration; + +use tokio::time; +use tonic::Request; +use tracing::{debug, error, info, warn}; + +use crate::db::{Database, DiskLayoutRow, DiskLayoutStatusRow}; +use crate::node_client::NodeClients; +use crate::node_proto; + +const RECONCILE_TICK: Duration = Duration::from_secs(15); +const NODE_RPC_TIMEOUT_SECS: i32 = 600; + +/// Spawn the DiskLayout reconciler loop in the background. +pub fn spawn_disk_layout_reconciler(db: Database, clients: NodeClients) { + tokio::spawn(async move { + let mut ticker = time::interval(RECONCILE_TICK); + // First tick fires immediately, so skip it to avoid a thundering herd + // on process start. + ticker.tick().await; + loop { + ticker.tick().await; + if let Err(e) = reconcile_once(&db, &clients).await { + warn!(error = %e, "disk layout reconcile tick failed"); + } + } + }); +} + +async fn reconcile_once(db: &Database, clients: &NodeClients) -> Result<(), String> { + let pending: Vec = db + .list_disk_layouts_needing_reconcile() + .map_err(|e| format!("list pending disk layouts: {e}"))?; + if pending.is_empty() { + return Ok(()); + } + debug!(count = pending.len(), "reconciling disk layouts"); + for row in pending { + if let Err(e) = reconcile_row(db, clients, &row).await { + warn!( + error = %e, + name = %row.name, + node_id = %row.node_id, + generation = row.generation, + "disk layout reconcile row failed" + ); + let _ = db.upsert_disk_layout_status(&DiskLayoutStatusRow { + name: row.name.clone(), + observed_generation: 0, + phase: "failed".to_string(), + refusal_reason: String::new(), + message: format!("controller reconcile error: {e}"), + last_transition_at: String::new(), + }); + } + } + Ok(()) +} + +async fn reconcile_row( + db: &Database, + clients: &NodeClients, + row: &DiskLayoutRow, +) -> Result<(), String> { + let node = db + .get_node(&row.node_id) + .map_err(|e| format!("lookup node {}: {e}", row.node_id))? + .ok_or_else(|| format!("node {} not registered", row.node_id))?; + let address = node.address.clone(); + if address.is_empty() { + return Err(format!("node {} has no address", row.node_id)); + } + + if clients.get_admin(&address).is_none() { + clients + .connect(&address) + .await + .map_err(|e| format!("connect to node {address}: {e}"))?; + } + let mut admin = clients + .get_admin(&address) + .ok_or_else(|| format!("no admin client for {address}"))?; + + let req = node_proto::ApplyDiskLayoutRequest { + disk_layout_nix: row.layout_nix.clone(), + apply: true, + timeout_seconds: NODE_RPC_TIMEOUT_SECS, + rebuild: true, + }; + let resp = admin + .apply_disk_layout(Request::new(req)) + .await + .map_err(|e| format!("apply_disk_layout rpc: {e}"))? + .into_inner(); + + let phase = if resp.success { + "applied" + } else if !resp.refusal_reason.is_empty() { + "refused" + } else { + "failed" + }; + db.upsert_disk_layout_status(&DiskLayoutStatusRow { + name: row.name.clone(), + observed_generation: row.generation, + phase: phase.to_string(), + refusal_reason: resp.refusal_reason.clone(), + message: resp.message.clone(), + last_transition_at: String::new(), + }) + .map_err(|e| format!("upsert status for {}: {e}", row.name))?; + + match phase { + "applied" => info!( + name = %row.name, + node_id = %row.node_id, + generation = row.generation, + "disk layout applied on node" + ), + "refused" => info!( + name = %row.name, + node_id = %row.node_id, + refusal_reason = %resp.refusal_reason, + "node-agent refused disk layout; operator must drain VMs and resubmit" + ), + _ => error!( + name = %row.name, + node_id = %row.node_id, + message = %resp.message, + "disk layout apply failed on node" + ), + } + Ok(()) +} diff --git a/crates/controller/src/grpc/controller.rs b/crates/controller/src/grpc/controller.rs index 18fb908..ffc3244 100644 --- a/crates/controller/src/grpc/controller.rs +++ b/crates/controller/src/grpc/controller.rs @@ -9,7 +9,8 @@ use crate::auth::{self, CN_CONTROLLER_PREFIX, CN_KCTL, CN_NODE_PREFIX}; use crate::config::{NetworkConfig, ReplicationConfig}; use crate::controller_proto; use crate::db::{ - Database, NetworkRow, NodeRow, SecurityGroupRow, SecurityGroupRuleRow, VmRow, WorkloadRow, + Database, DiskLayoutRow, DiskLayoutStatusRow, NetworkRow, NodeRow, SecurityGroupRow, + SecurityGroupRuleRow, VmRow, WorkloadRow, }; use crate::node_proto; use crate::{nixgen, node_client::NodeClients, scheduler}; @@ -47,6 +48,8 @@ const EVT_SECURITY_GROUP_DETACH: &str = "security_group.detach"; const EVT_NODE_DRAIN: &str = "node.drain"; const EVT_SSH_KEY_CREATE: &str = "ssh_key.create"; const EVT_SSH_KEY_DELETE: &str = "ssh_key.delete"; +const EVT_DISK_LAYOUT_CREATE: &str = "disk_layout.create"; +const EVT_DISK_LAYOUT_DELETE: &str = "disk_layout.delete"; fn normalize_sg_protocol(protocol: &str) -> Result { let p = protocol.trim().to_ascii_lowercase(); @@ -1005,6 +1008,305 @@ impl ControllerService { .map_err(|e| Status::internal(format!("append replication outbox row: {e}")))?; Ok(()) } + + // -- DiskLayout handlers ------------------------------------------------ + + async fn create_disk_layout_impl( + &self, + req: controller_proto::CreateDiskLayoutRequest, + ) -> Result, Status> { + let incoming = req + .disk_layout + .ok_or_else(|| Status::invalid_argument("disk_layout is required"))?; + let name = validate_network_name(&incoming.name)?; + let node_id = incoming.node_id.trim().to_string(); + if node_id.is_empty() { + return Err(Status::invalid_argument("disk_layout.node_id is required")); + } + // Node must exist; DiskLayouts target exactly one registered node. + if self + .db + .get_node(&node_id) + .map_err(|e| Status::internal(e.to_string()))? + .is_none() + { + return Err(Status::not_found(format!( + "node '{node_id}' is not registered" + ))); + } + let layout_nix = incoming.layout_nix.trim().to_string(); + if layout_nix.is_empty() { + return Err(Status::invalid_argument( + "disk_layout.layout_nix cannot be empty", + )); + } + // Minimal structural check: the DiskLayout still lowers to disko. + if !layout_nix.contains("disko.devices") { + return Err(Status::invalid_argument( + "disk_layout.layout_nix must define disko.devices", + )); + } + + let existing = self + .db + .get_disk_layout(&name) + .map_err(|e| Status::internal(e.to_string()))?; + + let (action, changed_fields, generation) = if let Some(existing) = existing.as_ref() { + if existing.node_id != node_id { + return Err(Status::invalid_argument(format!( + "cannot change immutable field(s) on disk layout '{name}': node_id \ + (delete the disk layout and recreate)" + ))); + } + if existing.layout_nix == layout_nix { + ( + controller_proto::ApplyAction::Unchanged as i32, + Vec::::new(), + existing.generation, + ) + } else { + ( + controller_proto::ApplyAction::Updated as i32, + vec!["layout_nix".to_string()], + existing.generation.saturating_add(1), + ) + } + } else { + ( + controller_proto::ApplyAction::Created as i32, + Vec::::new(), + 1, + ) + }; + + if action == controller_proto::ApplyAction::Unchanged as i32 { + let existing = existing.expect("unchanged implies existing"); + let status = self + .db + .get_disk_layout_status(&name) + .map_err(|e| Status::internal(e.to_string()))?; + let proto_layout = disk_layout_to_proto(&existing); + let _ = status; // status not returned on Create; Get exposes it + return Ok(Response::new(controller_proto::CreateDiskLayoutResponse { + success: true, + disk_layout: Some(proto_layout), + action, + changed_fields, + })); + } + + let row = DiskLayoutRow { + name: name.clone(), + node_id: node_id.clone(), + generation, + layout_nix: layout_nix.clone(), + created_at: String::new(), + updated_at: String::new(), + }; + let stored = self + .db + .upsert_disk_layout(&row) + .map_err(|e| Status::internal(format!("upserting disk layout: {e}")))?; + + // Reset status to Pending on any content change so the reconciler + // picks it up and the node-agent re-observes the new generation. + self.db + .upsert_disk_layout_status(&DiskLayoutStatusRow { + name: name.clone(), + observed_generation: 0, + phase: "pending".to_string(), + refusal_reason: String::new(), + message: String::new(), + last_transition_at: String::new(), + }) + .map_err(|e| Status::internal(format!("resetting disk layout status: {e}")))?; + + self.log_replication_event( + EVT_DISK_LAYOUT_CREATE, + &format!("disk-layout/{name}"), + serde_json::json!({ + "name": name, + "nodeId": node_id, + "generation": generation, + "layoutNix": layout_nix, + "action": match action { + x if x == controller_proto::ApplyAction::Created as i32 => "created", + x if x == controller_proto::ApplyAction::Updated as i32 => "updated", + _ => "unchanged", + }, + "changedFields": changed_fields, + }), + ); + + Ok(Response::new(controller_proto::CreateDiskLayoutResponse { + success: true, + disk_layout: Some(disk_layout_to_proto(&stored)), + action, + changed_fields, + })) + } + + async fn get_disk_layout_impl( + &self, + req: controller_proto::GetDiskLayoutRequest, + ) -> Result, Status> { + let name = req.name.trim(); + if name.is_empty() { + return Err(Status::invalid_argument("name is required")); + } + let row = self + .db + .get_disk_layout(name) + .map_err(|e| Status::internal(e.to_string()))? + .ok_or_else(|| Status::not_found(format!("disk layout '{name}' not found")))?; + let status = self + .db + .get_disk_layout_status(name) + .map_err(|e| Status::internal(e.to_string()))?; + Ok(Response::new(controller_proto::GetDiskLayoutResponse { + disk_layout: Some(disk_layout_to_proto(&row)), + status: status.map(|s| disk_layout_status_to_proto(&s)), + })) + } + + async fn list_disk_layouts_impl( + &self, + req: controller_proto::ListDiskLayoutsRequest, + ) -> Result, Status> { + let filter = req.node_id.trim(); + let rows = self + .db + .list_disk_layouts(if filter.is_empty() { + None + } else { + Some(filter) + }) + .map_err(|e| Status::internal(e.to_string()))?; + let statuses = self + .db + .list_disk_layout_statuses() + .map_err(|e| Status::internal(e.to_string()))?; + let status_by_name: HashMap = + statuses.into_iter().map(|s| (s.name.clone(), s)).collect(); + let summaries = rows + .iter() + .map(|r| controller_proto::DiskLayoutSummary { + disk_layout: Some(disk_layout_to_proto(r)), + status: status_by_name.get(&r.name).map(disk_layout_status_to_proto), + }) + .collect(); + Ok(Response::new(controller_proto::ListDiskLayoutsResponse { + disk_layouts: summaries, + })) + } + + async fn delete_disk_layout_impl( + &self, + req: controller_proto::DeleteDiskLayoutRequest, + ) -> Result, Status> { + let name = req.name.trim().to_string(); + if name.is_empty() { + return Err(Status::invalid_argument("name is required")); + } + let existed = self + .db + .delete_disk_layout(&name) + .map_err(|e| Status::internal(format!("deleting disk layout: {e}")))?; + if existed { + self.log_replication_event( + EVT_DISK_LAYOUT_DELETE, + &format!("disk-layout/{name}"), + serde_json::json!({"name": name}), + ); + } + Ok(Response::new(controller_proto::DeleteDiskLayoutResponse { + success: existed, + })) + } + + async fn classify_disk_layout_impl( + &self, + req: controller_proto::ClassifyDiskLayoutRequest, + ) -> Result, Status> { + // Controller-side pre-flight: cheap structural checks + target-device + // extraction. The authoritative verdict still comes from the + // node-agent classifier, which uses live lsblk state. + let layout = req + .disk_layout + .ok_or_else(|| Status::invalid_argument("disk_layout is required"))?; + let layout_nix = layout.layout_nix.trim(); + if layout_nix.is_empty() { + return Err(Status::invalid_argument( + "disk_layout.layout_nix cannot be empty", + )); + } + if !layout_nix.contains("disko.devices") { + return Ok(Response::new( + controller_proto::ClassifyDiskLayoutResponse { + safe: false, + refusal_reason: "invalid_layout".to_string(), + detail: "layout_nix must define disko.devices".to_string(), + target_devices: Vec::new(), + }, + )); + } + let target_devices = kcore_disko_types::extract_target_devices(layout_nix); + if target_devices.is_empty() { + return Ok(Response::new( + controller_proto::ClassifyDiskLayoutResponse { + safe: false, + refusal_reason: kcore_disko_types::refusal::NO_TARGET_DEVICES.to_string(), + detail: "layout did not declare any /dev/* target devices".to_string(), + target_devices, + }, + )); + } + // Controller does not yet maintain a replicated block-device inventory + // or a volume->device mapping table. Once those land (Phase 2 follow-up + // work) we will build an `LsblkSnapshot` here and call + // `kcore_disko_types::classify_disk_layout`. Until then, the + // controller's role is strictly structural: validate the manifest, + // surface the target devices for the operator to review, and let the + // node-agent have the last word. + Ok(Response::new( + controller_proto::ClassifyDiskLayoutResponse { + safe: true, + refusal_reason: String::new(), + detail: "controller pre-flight accepted; authoritative check runs on the node" + .to_string(), + target_devices, + }, + )) + } +} + +fn disk_layout_to_proto(row: &DiskLayoutRow) -> controller_proto::DiskLayout { + controller_proto::DiskLayout { + name: row.name.clone(), + node_id: row.node_id.clone(), + generation: row.generation, + layout_nix: row.layout_nix.clone(), + created_at: parse_datetime_to_timestamp(&row.created_at), + updated_at: parse_datetime_to_timestamp(&row.updated_at), + } +} + +fn disk_layout_status_to_proto(row: &DiskLayoutStatusRow) -> controller_proto::DiskLayoutStatus { + let phase = match row.phase.as_str() { + "pending" => controller_proto::DiskLayoutPhase::Pending as i32, + "applied" => controller_proto::DiskLayoutPhase::Applied as i32, + "refused" => controller_proto::DiskLayoutPhase::Refused as i32, + "failed" => controller_proto::DiskLayoutPhase::Failed as i32, + _ => controller_proto::DiskLayoutPhase::Unspecified as i32, + }; + controller_proto::DiskLayoutStatus { + observed_generation: row.observed_generation, + phase, + refusal_reason: row.refusal_reason.clone(), + message: row.message.clone(), + last_transition_at: parse_datetime_to_timestamp(&row.last_transition_at), + } } #[tonic::async_trait] @@ -4653,6 +4955,46 @@ impl controller_proto::controller_server::Controller for ControllerService { }, )) } + + async fn create_disk_layout( + &self, + request: Request, + ) -> Result, Status> { + auth::require_peer(&request, &[CN_KCTL, CN_CONTROLLER_PREFIX])?; + self.create_disk_layout_impl(request.into_inner()).await + } + + async fn get_disk_layout( + &self, + request: Request, + ) -> Result, Status> { + auth::require_peer(&request, &[CN_KCTL, CN_CONTROLLER_PREFIX])?; + self.get_disk_layout_impl(request.into_inner()).await + } + + async fn list_disk_layouts( + &self, + request: Request, + ) -> Result, Status> { + auth::require_peer(&request, &[CN_KCTL, CN_CONTROLLER_PREFIX])?; + self.list_disk_layouts_impl(request.into_inner()).await + } + + async fn delete_disk_layout( + &self, + request: Request, + ) -> Result, Status> { + auth::require_peer(&request, &[CN_KCTL, CN_CONTROLLER_PREFIX])?; + self.delete_disk_layout_impl(request.into_inner()).await + } + + async fn classify_disk_layout( + &self, + request: Request, + ) -> Result, Status> { + auth::require_peer(&request, &[CN_KCTL, CN_CONTROLLER_PREFIX])?; + self.classify_disk_layout_impl(request.into_inner()).await + } } fn acl(method: &str, identity: &str) -> controller_proto::AccessControlEntry { diff --git a/crates/controller/src/main.rs b/crates/controller/src/main.rs index 9542e25..626e218 100644 --- a/crates/controller/src/main.rs +++ b/crates/controller/src/main.rs @@ -9,6 +9,7 @@ mod auth; mod config; mod db; +mod disk_reconciler; mod grpc; mod nixgen; mod node_client; @@ -114,6 +115,7 @@ async fn main() -> anyhow::Result<()> { replication::spawn_compensation_executor(database.clone()); replication::spawn_head_materializer(database.clone()); replication::spawn_reservation_retry_executor(database.clone()); + disk_reconciler::spawn_disk_layout_reconciler(database.clone(), clients.clone()); let staleness_db = database.clone(); tokio::spawn(async move { diff --git a/crates/controller/src/replication.rs b/crates/controller/src/replication.rs index 977b1b9..4760d0f 100644 --- a/crates/controller/src/replication.rs +++ b/crates/controller/src/replication.rs @@ -634,6 +634,8 @@ fn apply_replication_event( | "security_group.detach" | "ssh_key.create" | "ssh_key.delete" + | "disk_layout.create" + | "disk_layout.delete" | "controller.register" => { // Phase-2 skeleton: payload validation + typed dispatch point. Ok(()) @@ -926,6 +928,13 @@ fn apply_domain_compensation( .map_err(|e| format!("compensate ssh_key.create delete {name}: {e}"))?; } } + "disk_layout.create" => { + if let Some(name) = loser_body.get("name").and_then(Value::as_str) { + let _ = db + .delete_disk_layout(name) + .map_err(|e| format!("compensate disk_layout.create delete {name}: {e}"))?; + } + } _ => {} } @@ -1570,6 +1579,30 @@ fn apply_head_to_domain(db: &Database, head: &ReplicationResourceHeadRow) -> Res .map_err(|e| format!("upsert controller peer {controller_id}: {e}"))?; Ok(()) } + "disk_layout.create" => { + let name = required_str(&body, "name", &head.resource_key)?; + let node_id = required_str(&body, "nodeId", &head.resource_key)?; + let layout_nix = required_str(&body, "layoutNix", &head.resource_key)?; + let generation = body.get("generation").and_then(Value::as_i64).unwrap_or(1); + let row = crate::db::DiskLayoutRow { + name: name.to_string(), + node_id: node_id.to_string(), + generation, + layout_nix: layout_nix.to_string(), + created_at: String::new(), + updated_at: String::new(), + }; + db.upsert_disk_layout(&row) + .map_err(|e| format!("upsert disk layout {name}: {e}"))?; + Ok(()) + } + "disk_layout.delete" => { + let name = required_str(&body, "name", &head.resource_key)?; + let _ = db + .delete_disk_layout(name) + .map_err(|e| format!("delete disk layout {name}: {e}"))?; + Ok(()) + } _ => Ok(()), } } diff --git a/crates/dashboard/src/app.rs b/crates/dashboard/src/app.rs index 88fc8d3..b4813f1 100644 --- a/crates/dashboard/src/app.rs +++ b/crates/dashboard/src/app.rs @@ -367,7 +367,7 @@ fn compliance_view(data: ComplianceDto) -> impl IntoView {

"Infrastructure"

"Controller"
{data.controller_version.clone()}
-
"OS"
"NixOS (declarative, atomic updates)"
+
"OS"
"kcoreOS (declarative, atomic updates)"
"Hypervisor"
"Cloud Hypervisor (KVM)"
"VM isolation"
"Hardware KVM + per-VM TAP + per-network bridge"
"Node approval"
"Required (approval queue)"
diff --git a/crates/dashboard/tests/pagination_test.rs b/crates/dashboard/tests/pagination_test.rs index ebfe5b1..b5ad4c8 100644 --- a/crates/dashboard/tests/pagination_test.rs +++ b/crates/dashboard/tests/pagination_test.rs @@ -360,6 +360,41 @@ impl Controller for ManyVmsController { ) -> Result, Status> { Err(Status::unimplemented("")) } + + async fn create_disk_layout( + &self, + _: TRequest, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + + async fn get_disk_layout( + &self, + _: TRequest, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + + async fn list_disk_layouts( + &self, + _: TRequest, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + + async fn delete_disk_layout( + &self, + _: TRequest, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + + async fn classify_disk_layout( + &self, + _: TRequest, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } } async fn spawn_many_vms_controller() -> SocketAddr { diff --git a/crates/dashboard/tests/support/mod.rs b/crates/dashboard/tests/support/mod.rs index d05335e..94e7929 100644 --- a/crates/dashboard/tests/support/mod.rs +++ b/crates/dashboard/tests/support/mod.rs @@ -444,6 +444,41 @@ impl Controller for MockController { ) -> Result, Status> { Err(unimp("list_volumes")) } + + async fn create_disk_layout( + &self, + _: Request, + ) -> Result, Status> { + Err(unimp("create_disk_layout")) + } + + async fn get_disk_layout( + &self, + _: Request, + ) -> Result, Status> { + Err(unimp("get_disk_layout")) + } + + async fn list_disk_layouts( + &self, + _: Request, + ) -> Result, Status> { + Err(unimp("list_disk_layouts")) + } + + async fn delete_disk_layout( + &self, + _: Request, + ) -> Result, Status> { + Err(unimp("delete_disk_layout")) + } + + async fn classify_disk_layout( + &self, + _: Request, + ) -> Result, Status> { + Err(unimp("classify_disk_layout")) + } } #[tonic::async_trait] diff --git a/crates/kcore-disko-types/Cargo.toml b/crates/kcore-disko-types/Cargo.toml new file mode 100644 index 0000000..09d826e --- /dev/null +++ b/crates/kcore-disko-types/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "kcore-disko-types" +version = "0.1.0" +edition = "2021" +description = "Shared, pure types for DiskLayout resources: target-device extractor, live state snapshot, and safe/dangerous classifier. std-only to keep Kani build time cheap." + +# Keep this crate dependency-free (std only). The node-agent and controller +# both depend on it and we want the Kani harness (phase4-kani) to compile in +# seconds. +[dependencies] + +[dev-dependencies] +proptest = "1.11" + +[lints] +workspace = true diff --git a/crates/kcore-disko-types/src/lib.rs b/crates/kcore-disko-types/src/lib.rs new file mode 100644 index 0000000..c5fd3e5 --- /dev/null +++ b/crates/kcore-disko-types/src/lib.rs @@ -0,0 +1,607 @@ +//! Shared types and pure logic for DiskLayout resources. +//! +//! Both the node-agent (authoritative classifier, fed by live `lsblk`) and +//! the controller (fast pre-flight, fed by replicated inventory) depend on +//! this crate so there is a single implementation of: +//! +//! * [`extract_target_devices`] — tokenise operator-supplied Nix to the set +//! of `/dev/...` paths the proposed layout would touch. +//! * [`classify_disk_layout`] — decide SAFE or DANGEROUS against an +//! [`LsblkSnapshot`]-shaped view of current disk state. +//! +//! The crate is deliberately dependency-free (std only) so the Kani harness +//! over the extractor compiles in seconds. + +#![forbid(unsafe_code)] + +use std::collections::BTreeSet; + +/// Stable, machine-readable refusal codes. Surfaced on the +/// `ApplyDiskLayoutResponse.refusal_reason` field so `kctl` can key UX off a +/// short string instead of parsing `message`. +pub mod refusal { + pub const MOUNTED_KCORE_VOLUME: &str = "target_device_has_active_kcore_volume"; + pub const MOUNTED_SYSTEM_PARTITION: &str = "target_device_has_active_system_mount"; + pub const ACTIVE_LVM_PV: &str = "target_device_is_active_lvm_pv"; + pub const ACTIVE_ZPOOL_MEMBER: &str = "target_device_is_active_zpool_member"; + pub const NO_TARGET_DEVICES: &str = "no_target_devices"; + pub const INVALID_LAYOUT: &str = "invalid_layout"; +} + +/// Snapshot of the block-device tree on the target node. On a node, this is +/// populated from `lsblk -J -o NAME,PATH,FSTYPE,MOUNTPOINTS,PKNAME,TYPE`; on +/// the controller, it is reconstructed from the replicated inventory. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct LsblkSnapshot { + pub devices: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BlockDevice { + /// Absolute path, e.g. `/dev/sda`, `/dev/sda1`, `/dev/mapper/cryptroot`. + pub path: String, + /// `disk`, `part`, `crypt`, `lvm`, `zfs_member`, etc. + pub kind: String, + /// Filesystem type (e.g. `ext4`, `LVM2_member`, `zfs_member`, empty). + pub fstype: Option, + /// Active mountpoints (if any). + pub mountpoints: Vec, + /// Parent device path (e.g. `/dev/sda` for `/dev/sda1`), if known. + pub parent_path: Option, +} + +/// Verdict returned by [`classify_disk_layout`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Verdict { + /// Safe to apply: no target device currently hosts active data. + Safe, + /// Refused: the proposed layout would destroy active state. + Dangerous { + /// Stable refusal code (one of [`refusal`]). + code: &'static str, + /// Human-readable explanation suitable for the operator. + detail: String, + }, +} + +/// Classify a proposed disk layout. +/// +/// * `target_devices` — the block-device paths referenced by `device = "..."` +/// attributes in the proposed layout, as extracted by +/// [`extract_target_devices`]. +/// * `snapshot` — live lsblk snapshot of the node. +/// * `kcore_volume_roots` — mountpoint prefixes the node uses for +/// workload-backing volumes (e.g. `/var/lib/kcore/volumes`). +pub fn classify_disk_layout( + target_devices: &[String], + snapshot: &LsblkSnapshot, + kcore_volume_roots: &[&str], +) -> Verdict { + if target_devices.is_empty() { + return Verdict::Dangerous { + code: refusal::NO_TARGET_DEVICES, + detail: "proposed layout did not declare any /dev/* target devices".to_string(), + }; + } + + let target_set: BTreeSet<&str> = target_devices.iter().map(|s| s.as_str()).collect(); + + for dev in &snapshot.devices { + if !is_within_targets(&dev.path, dev.parent_path.as_deref(), &target_set, snapshot) { + continue; + } + + if let Some(danger) = classify_device(dev, kcore_volume_roots) { + return danger; + } + } + + Verdict::Safe +} + +fn classify_device(dev: &BlockDevice, kcore_volume_roots: &[&str]) -> Option { + for mp in &dev.mountpoints { + if mp.is_empty() { + continue; + } + if kcore_volume_roots.iter().any(|root| mp.starts_with(root)) { + return Some(Verdict::Dangerous { + code: refusal::MOUNTED_KCORE_VOLUME, + detail: format!("{} currently backs kcore volume mount {}", dev.path, mp), + }); + } + if is_system_mount(mp) { + return Some(Verdict::Dangerous { + code: refusal::MOUNTED_SYSTEM_PARTITION, + detail: format!("{} currently hosts system mount {}", dev.path, mp), + }); + } + } + + match dev.fstype.as_deref() { + Some("LVM2_member") => Some(Verdict::Dangerous { + code: refusal::ACTIVE_LVM_PV, + detail: format!("{} is an active LVM physical volume", dev.path), + }), + Some("zfs_member") => Some(Verdict::Dangerous { + code: refusal::ACTIVE_ZPOOL_MEMBER, + detail: format!("{} is a member of an active ZFS pool", dev.path), + }), + _ => None, + } +} + +fn is_system_mount(mp: &str) -> bool { + matches!(mp, "/" | "/boot" | "/nix" | "/nix/store" | "/boot/efi") +} + +fn is_within_targets( + path: &str, + parent: Option<&str>, + targets: &BTreeSet<&str>, + snapshot: &LsblkSnapshot, +) -> bool { + if targets.contains(path) { + return true; + } + let mut cursor = parent; + let mut depth = 0usize; + while let Some(p) = cursor { + if depth > 8 { + return false; + } + if targets.contains(p) { + return true; + } + cursor = snapshot + .devices + .iter() + .find(|d| d.path == p) + .and_then(|d| d.parent_path.as_deref()); + depth += 1; + } + false +} + +/// Extract `/dev/...` device paths that appear as the right-hand side of +/// `device = "..."` assignments in the proposed disko expression. +/// +/// This is a deliberately lenient string parse — full Nix evaluation is +/// expensive and happens later via `disko`. We only need enough accuracy to +/// enumerate the disks the operator intends to partition. +/// +/// Line comments (`# ...`) and block comments (`/* ... */`) are skipped so +/// commented-out `device = "..."` lines don't leak into the target set. +pub fn extract_target_devices(disk_layout_nix: &str) -> Vec { + let sanitized = strip_nix_comments(disk_layout_nix); + let mut out = Vec::new(); + let bytes = sanitized.as_bytes(); + let key = b"device"; + let mut i = 0usize; + while i + key.len() < bytes.len() { + if &bytes[i..i + key.len()] == key { + let prev_ok = i == 0 || { + let p = bytes[i - 1]; + !(p.is_ascii_alphanumeric() || p == b'_') + }; + if !prev_ok { + i += 1; + continue; + } + let mut j = i + key.len(); + while j < bytes.len() && (bytes[j] == b' ' || bytes[j] == b'\t') { + j += 1; + } + if j < bytes.len() && bytes[j] == b'=' { + j += 1; + while j < bytes.len() && (bytes[j] == b' ' || bytes[j] == b'\t') { + j += 1; + } + if j < bytes.len() && bytes[j] == b'"' { + j += 1; + let start = j; + while j < bytes.len() && bytes[j] != b'"' { + j += 1; + } + if j <= bytes.len() { + let val = &sanitized[start..j]; + if val.starts_with("/dev/") { + out.push(val.to_string()); + } + } + } + } + i = j.max(i + 1); + continue; + } + i += 1; + } + out.sort(); + out.dedup(); + out +} + +/// Replace Nix line and block comments with spaces of equal length, preserving +/// offsets into the original string. Strings (`"..."`) are left intact so +/// `#` characters inside string literals do not start a comment. +fn strip_nix_comments(src: &str) -> String { + let bytes = src.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0usize; + while i < bytes.len() { + let b = bytes[i]; + if b == b'"' { + out.push(b); + i += 1; + while i < bytes.len() { + let c = bytes[i]; + out.push(c); + i += 1; + if c == b'\\' && i < bytes.len() { + out.push(bytes[i]); + i += 1; + continue; + } + if c == b'"' { + break; + } + } + continue; + } + if b == b'#' { + while i < bytes.len() && bytes[i] != b'\n' { + out.push(b' '); + i += 1; + } + continue; + } + if b == b'/' && i + 1 < bytes.len() && bytes[i + 1] == b'*' { + out.push(b' '); + out.push(b' '); + i += 2; + while i + 1 < bytes.len() && !(bytes[i] == b'*' && bytes[i + 1] == b'/') { + out.push(if bytes[i] == b'\n' { b'\n' } else { b' ' }); + i += 1; + } + if i + 1 < bytes.len() { + out.push(b' '); + out.push(b' '); + i += 2; + } + continue; + } + out.push(b); + i += 1; + } + String::from_utf8(out).unwrap_or_else(|_| src.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn disk( + path: &str, + kind: &str, + fstype: Option<&str>, + mp: &[&str], + parent: Option<&str>, + ) -> BlockDevice { + BlockDevice { + path: path.to_string(), + kind: kind.to_string(), + fstype: fstype.map(str::to_string), + mountpoints: mp.iter().map(|s| s.to_string()).collect(), + parent_path: parent.map(str::to_string), + } + } + + #[test] + fn empty_layout_is_dangerous_because_no_targets() { + let snap = LsblkSnapshot::default(); + let verdict = classify_disk_layout(&[], &snap, &["/var/lib/kcore/volumes"]); + assert!(matches!(verdict, Verdict::Dangerous { .. })); + } + + #[test] + fn idle_disk_is_safe() { + let snap = LsblkSnapshot { + devices: vec![disk("/dev/sdb", "disk", None, &[], None)], + }; + let verdict = classify_disk_layout( + &["/dev/sdb".to_string()], + &snap, + &["/var/lib/kcore/volumes"], + ); + assert_eq!(verdict, Verdict::Safe); + } + + #[test] + fn target_with_kcore_volume_mount_is_dangerous() { + let snap = LsblkSnapshot { + devices: vec![ + disk("/dev/sdc", "disk", None, &[], None), + disk( + "/dev/sdc1", + "part", + Some("ext4"), + &["/var/lib/kcore/volumes"], + Some("/dev/sdc"), + ), + ], + }; + let verdict = classify_disk_layout( + &["/dev/sdc".to_string()], + &snap, + &["/var/lib/kcore/volumes"], + ); + match verdict { + Verdict::Dangerous { code, .. } => assert_eq!(code, refusal::MOUNTED_KCORE_VOLUME), + other => panic!("expected Dangerous, got {other:?}"), + } + } + + #[test] + fn extract_target_devices_handles_multiple_and_comments() { + let nix = r#"{ + disko.devices.disk.os = { + device = "/dev/sda"; # the OS disk + content.type = "gpt"; + }; + disko.devices.disk.data0.device = "/dev/nvme0n1"; + # fake out: device = "/dev/should_ignore" inside a comment + }"#; + let got = extract_target_devices(nix); + assert_eq!( + got, + vec!["/dev/nvme0n1".to_string(), "/dev/sda".to_string()] + ); + } + + #[test] + fn extract_target_devices_skips_block_comments() { + let nix = r#" + /* device = "/dev/inside_block_comment"; */ + device = "/dev/real"; + "#; + assert_eq!(extract_target_devices(nix), vec!["/dev/real".to_string()]); + } + + #[test] + fn extract_target_devices_keeps_hash_inside_strings() { + let nix = r#"device = "/dev/disk/by-id/pool#foo";"#; + assert_eq!( + extract_target_devices(nix), + vec!["/dev/disk/by-id/pool#foo".to_string()] + ); + } +} + +#[cfg(test)] +mod prop_tests { + use super::*; + use proptest::collection::vec as proptest_vec; + use proptest::prelude::*; + + /// A non-pathological device name segment. + fn device_path() -> impl Strategy { + "[a-z][a-z0-9]{1,8}".prop_map(|s| format!("/dev/{s}")) + } + + fn arb_block_device(parent: Option) -> impl Strategy { + ( + device_path(), + prop_oneof![Just("disk"), Just("part"), Just("lvm"), Just("crypt")], + prop_oneof![ + Just(None), + Just(Some("ext4".to_string())), + Just(Some("LVM2_member".to_string())), + Just(Some("zfs_member".to_string())) + ], + proptest_vec( + prop_oneof![ + Just("".to_string()), + Just("/".to_string()), + Just("/boot".to_string()), + Just("/var/lib/kcore/volumes/v1".to_string()), + Just("/srv/data".to_string()) + ], + 0..3, + ), + ) + .prop_map(move |(path, kind, fstype, mps)| BlockDevice { + path, + kind: kind.to_string(), + fstype, + mountpoints: mps.into_iter().filter(|s| !s.is_empty()).collect(), + parent_path: parent.clone(), + }) + } + + fn arb_snapshot() -> impl Strategy { + proptest_vec(arb_block_device(None), 0..12).prop_map(|devices| LsblkSnapshot { devices }) + } + + proptest! { + /// Safety invariant. If [`classify_disk_layout`] returns [`Verdict::Safe`], + /// then no device in the snapshot that is "in target scope" (the device + /// itself is a target, or transitively descends from one) currently has: + /// * a mountpoint under any kcore volume root + /// * a system mountpoint + /// * fstype == LVM2_member or zfs_member + /// + /// This is the central anti-foot-gun guarantee operators rely on: + /// "SAFE" must mean "no active VM-backing storage will be wiped". + #[test] + fn safe_never_touches_active_storage( + targets in proptest_vec(device_path(), 1..6), + snap in arb_snapshot(), + ) { + let roots: &[&str] = &["/var/lib/kcore/volumes"]; + let verdict = classify_disk_layout(&targets, &snap, roots); + if let Verdict::Safe = verdict { + let target_set: std::collections::BTreeSet<&str> = + targets.iter().map(|s| s.as_str()).collect(); + for dev in &snap.devices { + let in_scope = is_within_targets( + &dev.path, + dev.parent_path.as_deref(), + &target_set, + &snap, + ); + if !in_scope { + continue; + } + for mp in &dev.mountpoints { + prop_assert!( + !roots.iter().any(|r| mp.starts_with(r)), + "SAFE verdict but {} is mounted at kcore volume root {}", + dev.path, + mp + ); + prop_assert!( + !matches!(mp.as_str(), "/" | "/boot" | "/nix" | "/nix/store" | "/boot/efi"), + "SAFE verdict but {} hosts system mount {}", + dev.path, + mp + ); + } + if let Some(fs) = dev.fstype.as_deref() { + prop_assert!( + fs != "LVM2_member" && fs != "zfs_member", + "SAFE verdict but {} is an active {} member", + dev.path, + fs + ); + } + } + } + } + + /// Determinism: classification depends only on inputs. + #[test] + fn classifier_is_deterministic( + targets in proptest_vec(device_path(), 1..6), + snap in arb_snapshot(), + ) { + let roots: &[&str] = &["/var/lib/kcore/volumes"]; + let a = classify_disk_layout(&targets, &snap, roots); + let b = classify_disk_layout(&targets, &snap, roots); + prop_assert_eq!(a, b); + } + + /// `extract_target_devices` is idempotent: extracting from a layout we + /// rebuild from its own targets reproduces the same set. + #[test] + fn extract_target_devices_roundtrip(paths in proptest_vec(device_path(), 0..6)) { + let body = paths.iter() + .map(|p| format!("device = \"{p}\";")) + .collect::>() + .join("\n"); + let mut expected: Vec = paths.clone(); + expected.sort(); + expected.dedup(); + let got = extract_target_devices(&body); + prop_assert_eq!(got, expected); + } + } +} + +// ============================================================= +// Bounded model-checking proofs (Phase 4 — Kani) +// ============================================================= +// +// The layout-diff parser (`extract_target_devices` + the +// `strip_nix_comments` helper it depends on) is the single piece +// of logic that turns operator-supplied Nix text into the set of +// `/dev/...` paths the controller and node-agent reason about. +// A bug here can either: +// * silently *miss* a target device, causing the classifier to +// greenlight a layout that actually wipes an active disk +// (a soundness hole in the safety story), or +// * blow up with a panic / out-of-bounds read on adversarial +// input (a denial-of-service for the controller RPC). +// +// These harnesses are the formal complement to the proptest +// fuzzer in `prop_tests` above: proptest looks for failing +// inputs by sampling, Kani exhaustively explores every byte +// pattern up to `MAX_INPUT_LEN`. The bound is small on purpose +// so the proof finishes in seconds and slots into the existing +// per-harness CI matrix in `.github/workflows/formal-checks.yml`. +// +// Run locally with: +// +// ```text +// cargo install --locked kani-verifier +// cargo kani setup +// cargo kani -p kcore-disko-types +// ``` +#[cfg(kani)] +mod kani_proofs { + use super::*; + + const MAX_INPUT_LEN: usize = 8; + + fn any_ascii_str(buf: &mut [u8; MAX_INPUT_LEN]) -> &str { + let len: usize = kani::any(); + kani::assume(len <= MAX_INPUT_LEN); + for slot in buf.iter_mut() { + let b: u8 = kani::any(); + kani::assume(b < 128); + *slot = b; + } + std::str::from_utf8(&buf[..len]).unwrap() + } + + /// **Liveness**: `extract_target_devices` never panics on any + /// ASCII input up to the bound. This rules out a panicking + /// controller-side parser feeding adversarial layout bodies. + #[kani::proof] + #[kani::unwind(10)] + fn extract_target_devices_never_panics() { + let mut buf = [0u8; MAX_INPUT_LEN]; + let s = any_ascii_str(&mut buf); + let _ = extract_target_devices(s); + } + + /// **Liveness**: the comment stripper never panics either, and + /// preserves length. Length preservation is what lets the + /// extractor's byte indices stay valid against the original + /// source text. + #[kani::proof] + #[kani::unwind(10)] + fn strip_nix_comments_preserves_length() { + let mut buf = [0u8; MAX_INPUT_LEN]; + let s = any_ascii_str(&mut buf); + let stripped = strip_nix_comments(s); + assert!(stripped.len() == s.len()); + } + + /// **Soundness shape**: every device path the extractor returns + /// starts with `/dev/`. The classifier's `is_within_targets` + /// check assumes this prefix; if the parser ever returned + /// `"…something else…"` the classifier could silently no-op. + #[kani::proof] + #[kani::unwind(10)] + fn extract_target_devices_outputs_dev_prefixed_paths() { + let mut buf = [0u8; MAX_INPUT_LEN]; + let s = any_ascii_str(&mut buf); + let out = extract_target_devices(s); + for p in &out { + assert!(p.starts_with("/dev/")); + } + } + + /// **Determinism**: extracting twice yields the same set. This + /// catches any accidental dependence on uninitialised memory, + /// hash randomisation, or iterator order. + #[kani::proof] + #[kani::unwind(10)] + fn extract_target_devices_is_deterministic() { + let mut buf = [0u8; MAX_INPUT_LEN]; + let s = any_ascii_str(&mut buf); + let a = extract_target_devices(s); + let b = extract_target_devices(s); + assert!(a == b); + } +} diff --git a/crates/kctl/src/commands/apply.rs b/crates/kctl/src/commands/apply.rs index 94bb2d7..e18754e 100644 --- a/crates/kctl/src/commands/apply.rs +++ b/crates/kctl/src/commands/apply.rs @@ -1,5 +1,5 @@ use crate::client::{self, controller_proto}; -use crate::commands::{container, network, security_group, ssh_key, vm}; +use crate::commands::{container, disk_layout, network, security_group, ssh_key, vm}; use crate::config::ConnectionInfo; use anyhow::{Context, Result}; @@ -88,6 +88,9 @@ pub async fn apply(info: &ConnectionInfo, file: &str, dry_run: bool) -> Result<( return ssh_key::create_from_manifest(info, file).await } "container" => return container::create_from_manifest(info, file).await, + "disklayout" | "disk-layout" | "disk_layout" => { + return disk_layout::apply_from_file(info, file).await + } _ => {} } } diff --git a/crates/kctl/src/commands/cluster.rs b/crates/kctl/src/commands/cluster.rs index 230e6d6..27867a9 100644 --- a/crates/kctl/src/commands/cluster.rs +++ b/crates/kctl/src/commands/cluster.rs @@ -47,7 +47,7 @@ pub fn create_from_manifest(file: &str, config_path: &Path) -> Result<()> { PathBuf::from(&p) } }) - .unwrap_or_else(|| config::default_cluster_certs_dir(&context_name)); + .unwrap_or_else(|| default_cluster_certs_dir_with_fallback(&context_name, config_path)); create( config_path, @@ -58,6 +58,20 @@ pub fn create_from_manifest(file: &str, config_path: &Path) -> Result<()> { ) } +fn default_cluster_certs_dir_with_fallback(context_name: &str, config_path: &Path) -> PathBuf { + let preferred = config::default_cluster_certs_dir(context_name); + if std::fs::create_dir_all(&preferred).is_ok() { + return preferred; + } + + let fallback_root = config_path + .parent() + .map(Path::to_path_buf) + .unwrap_or_else(|| PathBuf::from(".")) + .join(".kcore"); + fallback_root.join(context_name) +} + pub fn create( config_path: &Path, controller: &str, diff --git a/crates/kctl/src/commands/disk_layout.rs b/crates/kctl/src/commands/disk_layout.rs new file mode 100644 index 0000000..15ae60a --- /dev/null +++ b/crates/kctl/src/commands/disk_layout.rs @@ -0,0 +1,345 @@ +//! `kctl` subcommands for the [`DiskLayout`] controller resource. +//! +//! The manifest surface looks like: +//! +//! ```yaml +//! kind: DiskLayout +//! metadata: +//! name: datastore-sata +//! spec: +//! nodeId: node-prod-01 +//! layoutNix: | +//! { disko.devices = { disk.data = { device = "/dev/sda"; ... }; }; } +//! ``` +//! +//! `layoutNixFile` is accepted as a shortcut to read the Nix body from a +//! file next to the manifest, so operators don't have to inline a +//! several-KB-long expression. + +use std::path::Path; + +use anyhow::{bail, Context, Result}; +use serde::Deserialize; + +use crate::apply_summary::render_apply_summary; +use crate::client::{self, controller_proto}; +use crate::config::ConnectionInfo; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct DiskLayoutManifest { + kind: String, + metadata: ManifestMetadata, + spec: DiskLayoutSpec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ManifestMetadata { + name: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct DiskLayoutSpec { + node_id: String, + #[serde(default)] + layout_nix: String, + #[serde(default)] + layout_nix_file: String, +} + +pub async fn apply_from_file(info: &ConnectionInfo, file: &str) -> Result<()> { + let manifest = parse_manifest(file)?; + let layout_nix = resolve_layout_nix(file, &manifest.spec)?; + if !layout_nix.contains("disko.devices") { + bail!( + "disk layout '{}' must define disko.devices in its layoutNix body", + manifest.metadata.name + ); + } + + let mut client = client::controller_client(info).await?; + let resp = client + .create_disk_layout(controller_proto::CreateDiskLayoutRequest { + disk_layout: Some(controller_proto::DiskLayout { + name: manifest.metadata.name.clone(), + node_id: manifest.spec.node_id.trim().to_string(), + generation: 0, // server assigns + layout_nix, + created_at: None, + updated_at: None, + }), + }) + .await + .context("create_disk_layout rpc")? + .into_inner(); + + let label = format!("disk layout '{}'", manifest.metadata.name); + println!( + "{}", + render_apply_summary(resp.action, &resp.changed_fields, &label) + ); + Ok(()) +} + +pub async fn diff_from_file(info: &ConnectionInfo, file: &str) -> Result<()> { + let manifest = parse_manifest(file)?; + let layout_nix = resolve_layout_nix(file, &manifest.spec)?; + + let mut client = client::controller_client(info).await?; + let resp = client + .classify_disk_layout(controller_proto::ClassifyDiskLayoutRequest { + disk_layout: Some(controller_proto::DiskLayout { + name: manifest.metadata.name.clone(), + node_id: manifest.spec.node_id.trim().to_string(), + generation: 0, + layout_nix, + created_at: None, + updated_at: None, + }), + }) + .await + .context("classify_disk_layout rpc")? + .into_inner(); + + println!("Disk layout: {}", manifest.metadata.name); + println!("Target node: {}", manifest.spec.node_id); + println!("Target disks:"); + if resp.target_devices.is_empty() { + println!(" (none detected)"); + } else { + for d in &resp.target_devices { + println!(" - {d}"); + } + } + if resp.safe { + println!("Pre-flight: SAFE"); + } else { + println!("Pre-flight: REFUSED ({})", resp.refusal_reason); + } + if !resp.detail.is_empty() { + println!("Detail: {}", resp.detail); + } + println!( + "Note: controller pre-flight is advisory. The node-agent classifier \ + runs against live lsblk state and has the final say on every apply." + ); + Ok(()) +} + +pub async fn list(info: &ConnectionInfo, node_filter: Option<&str>) -> Result<()> { + let mut client = client::controller_client(info).await?; + let resp = client + .list_disk_layouts(controller_proto::ListDiskLayoutsRequest { + node_id: node_filter.unwrap_or_default().to_string(), + }) + .await? + .into_inner(); + if resp.disk_layouts.is_empty() { + println!("No disk layouts found"); + return Ok(()); + } + println!( + "{:<24} {:<24} {:>4} {:<10} {:<20}", + "NAME", "NODE", "GEN", "PHASE", "REFUSAL_REASON" + ); + for entry in resp.disk_layouts { + let layout = entry.disk_layout.unwrap_or_default(); + let status = entry.status.unwrap_or_default(); + println!( + "{:<24} {:<24} {:>4} {:<10} {:<20}", + layout.name, + layout.node_id, + layout.generation, + phase_str(status.phase), + status.refusal_reason, + ); + } + Ok(()) +} + +pub async fn get(info: &ConnectionInfo, name: &str) -> Result<()> { + let mut client = client::controller_client(info).await?; + let resp = client + .get_disk_layout(controller_proto::GetDiskLayoutRequest { + name: name.to_string(), + }) + .await? + .into_inner(); + let layout = resp.disk_layout.context("disk layout not found")?; + println!("Name: {}", layout.name); + println!("Node: {}", layout.node_id); + println!("Generation: {}", layout.generation); + if let Some(status) = resp.status { + println!("Observed: {}", status.observed_generation); + println!("Phase: {}", phase_str(status.phase)); + if !status.refusal_reason.is_empty() { + println!("Refusal: {}", status.refusal_reason); + } + if !status.message.is_empty() { + println!("Message: {}", status.message); + } + } + println!("Layout (Nix):"); + for line in layout.layout_nix.lines() { + println!(" {line}"); + } + Ok(()) +} + +pub async fn delete(info: &ConnectionInfo, name: &str) -> Result<()> { + let mut client = client::controller_client(info).await?; + let resp = client + .delete_disk_layout(controller_proto::DeleteDiskLayoutRequest { + name: name.to_string(), + }) + .await? + .into_inner(); + if resp.success { + println!("Deleted disk layout '{name}'"); + } else { + println!("Disk layout '{name}' not found"); + } + Ok(()) +} + +fn parse_manifest(file: &str) -> Result { + let content = std::fs::read_to_string(file).with_context(|| format!("reading {file}"))?; + let manifest: DiskLayoutManifest = serde_yaml::from_str(&content) + .with_context(|| format!("parsing manifest YAML in {file}"))?; + if !manifest.kind.eq_ignore_ascii_case("DiskLayout") { + bail!( + "expected kind: DiskLayout in {file}, got kind: {}", + manifest.kind + ); + } + if manifest.metadata.name.trim().is_empty() { + bail!("metadata.name is required in {file}"); + } + if manifest.spec.node_id.trim().is_empty() { + bail!("spec.nodeId is required in {file}"); + } + Ok(manifest) +} + +fn resolve_layout_nix(manifest_path: &str, spec: &DiskLayoutSpec) -> Result { + let has_inline = !spec.layout_nix.trim().is_empty(); + let has_file = !spec.layout_nix_file.trim().is_empty(); + match (has_inline, has_file) { + (true, true) => { + bail!("spec.layoutNix and spec.layoutNixFile are mutually exclusive; pick one") + } + (true, false) => Ok(spec.layout_nix.clone()), + (false, true) => { + let base = Path::new(manifest_path) + .parent() + .unwrap_or_else(|| Path::new(".")); + let full = base.join(spec.layout_nix_file.trim()); + std::fs::read_to_string(&full) + .with_context(|| format!("reading layoutNixFile {}", full.display())) + } + (false, false) => bail!("one of spec.layoutNix or spec.layoutNixFile is required"), + } +} + +fn phase_str(phase: i32) -> &'static str { + match controller_proto::DiskLayoutPhase::try_from(phase) + .unwrap_or(controller_proto::DiskLayoutPhase::Unspecified) + { + controller_proto::DiskLayoutPhase::Unspecified => "unspecified", + controller_proto::DiskLayoutPhase::Pending => "pending", + controller_proto::DiskLayoutPhase::Applied => "applied", + controller_proto::DiskLayoutPhase::Refused => "refused", + controller_proto::DiskLayoutPhase::Failed => "failed", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn manifest_parses_with_inline_nix() { + let manifest = r#" +kind: DiskLayout +metadata: + name: ssd-pool +spec: + nodeId: node-a + layoutNix: | + { disko.devices = { disk.data = { device = "/dev/sda"; }; }; } +"#; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("dl.yaml"); + std::fs::write(&path, manifest).unwrap(); + let got = parse_manifest(path.to_str().unwrap()).unwrap(); + assert_eq!(got.metadata.name, "ssd-pool"); + assert_eq!(got.spec.node_id, "node-a"); + assert!(got.spec.layout_nix.contains("disko.devices")); + } + + #[test] + fn manifest_rejects_wrong_kind() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("dl.yaml"); + std::fs::write( + &path, + "kind: VM\nmetadata:\n name: x\nspec:\n nodeId: n\n", + ) + .unwrap(); + let err = parse_manifest(path.to_str().unwrap()).unwrap_err(); + assert!(format!("{err:#}").contains("expected kind: DiskLayout")); + } + + #[test] + fn manifest_requires_node_id() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("dl.yaml"); + std::fs::write( + &path, + "kind: DiskLayout\nmetadata:\n name: x\nspec:\n nodeId: \"\"\n layoutNix: \"disko.devices = {};\"\n", + ) + .unwrap(); + let err = parse_manifest(path.to_str().unwrap()).unwrap_err(); + assert!(format!("{err:#}").contains("spec.nodeId is required")); + } + + #[test] + fn resolve_requires_one_of_inline_or_file() { + let spec = DiskLayoutSpec { + node_id: "n".to_string(), + layout_nix: String::new(), + layout_nix_file: String::new(), + }; + let err = resolve_layout_nix("/tmp/does-not-exist", &spec).unwrap_err(); + assert!(format!("{err:#}").contains("one of spec.layoutNix")); + } + + #[test] + fn resolve_refuses_both_inline_and_file() { + let spec = DiskLayoutSpec { + node_id: "n".to_string(), + layout_nix: "disko.devices = {};".to_string(), + layout_nix_file: "disk.nix".to_string(), + }; + let err = resolve_layout_nix("/tmp/does-not-exist", &spec).unwrap_err(); + assert!(format!("{err:#}").contains("mutually exclusive")); + } + + #[test] + fn resolve_reads_file_next_to_manifest() { + let dir = tempfile::tempdir().unwrap(); + let nix_path = dir.path().join("disk.nix"); + std::fs::write(&nix_path, "{ disko.devices = {}; }").unwrap(); + let manifest_path = dir.path().join("dl.yaml"); + std::fs::write(&manifest_path, "ignored").unwrap(); + let spec = DiskLayoutSpec { + node_id: "n".to_string(), + layout_nix: String::new(), + layout_nix_file: "disk.nix".to_string(), + }; + let got = resolve_layout_nix(manifest_path.to_str().unwrap(), &spec).unwrap(); + assert!(got.contains("disko.devices")); + } +} diff --git a/crates/kctl/src/commands/mod.rs b/crates/kctl/src/commands/mod.rs index 536a7cd..c084917 100644 --- a/crates/kctl/src/commands/mod.rs +++ b/crates/kctl/src/commands/mod.rs @@ -4,6 +4,7 @@ pub mod cluster; pub mod compliance; pub mod conflict; pub mod container; +pub mod disk_layout; pub mod image; pub mod network; pub mod node; diff --git a/crates/kctl/src/commands/node.rs b/crates/kctl/src/commands/node.rs index 9ddf60c..82ca0c7 100644 --- a/crates/kctl/src/commands/node.rs +++ b/crates/kctl/src/commands/node.rs @@ -681,28 +681,41 @@ pub async fn apply_nix(info: &ConnectionInfo, file: &str, rebuild: bool) -> Resu } } -pub async fn apply_disko_layout( +pub async fn apply_disk_layout( info: &ConnectionInfo, file: &str, apply: bool, timeout_seconds: i32, + rebuild: bool, ) -> Result<()> { let content = std::fs::read_to_string(file).with_context(|| format!("reading {file}"))?; let mut client = client::node_admin_client(info).await?; let resp = client - .apply_disko_layout(node_proto::ApplyDiskoLayoutRequest { - disko_nix: content, + .apply_disk_layout(node_proto::ApplyDiskLayoutRequest { + disk_layout_nix: content, apply, timeout_seconds, + rebuild, }) .await? .into_inner(); if resp.success { println!("{}", resp.message); - println!("disko mode: {}", resp.mode); + println!("disk management mode: {}", resp.mode); Ok(()) + } else if !resp.refusal_reason.is_empty() { + anyhow::bail!( + "disk layout apply refused (mode={}, reason={}): {}", + resp.mode, + resp.refusal_reason, + resp.message + ) } else { - anyhow::bail!("disko apply failed (mode={}): {}", resp.mode, resp.message) + anyhow::bail!( + "disk layout apply failed (mode={}): {}", + resp.mode, + resp.message + ) } } diff --git a/crates/kctl/src/main.rs b/crates/kctl/src/main.rs index 83333c4..f3dea38 100644 --- a/crates/kctl/src/main.rs +++ b/crates/kctl/src/main.rs @@ -146,6 +146,14 @@ enum Command { #[arg(long)] dry_run: bool, }, + /// Show a controller-side pre-flight diff/safety report for a manifest. + /// Currently supports `kind: DiskLayout` manifests; other kinds will be + /// added as their classifiers land. + Diff { + /// Path to the manifest file + #[arg(short = 'f', long = "filename")] + file: String, + }, /// Show version Version, /// Unified workload operations via controller API @@ -355,6 +363,12 @@ enum DeleteResource { #[arg(long)] force: bool, }, + /// Delete a DiskLayout (controller-side; does not touch the node) + #[command(name = "disk-layout", alias = "disklayout")] + DiskLayout { + /// DiskLayout name + name: String, + }, } #[derive(Subcommand)] @@ -482,6 +496,13 @@ enum GetResource { #[arg(long)] require_healthy: bool, }, + /// List DiskLayout resources + #[command(name = "disk-layouts", alias = "disk-layout", alias = "disklayout")] + DiskLayouts { + /// Optional node filter (only show layouts targeted at this node) + #[arg(long = "target-node")] + target_node: Option, + }, } #[derive(Subcommand)] @@ -525,6 +546,12 @@ enum DescribeResource { /// Describe cluster compliance report #[command(name = "compliance-report", alias = "compliance")] ComplianceReport, + /// Describe a DiskLayout (current generation, observed status, body) + #[command(name = "disk-layout", alias = "disklayout")] + DiskLayout { + /// DiskLayout name + name: String, + }, } #[derive(Subcommand)] @@ -597,9 +624,10 @@ enum NodeAction { #[arg(long)] no_rebuild: bool, }, - /// Validate/apply a day-2 disko layout on a node - ApplyDisko { - /// Path to disko Nix expression file + /// Validate/apply a day-2 disk layout on a node (built on disko) + #[command(visible_alias = "apply-disko")] + ApplyDisk { + /// Path to the disk-layout Nix expression file #[arg(short = 'f', long = "filename")] file: String, /// Apply partitioning/mount changes (default validates only) @@ -608,6 +636,9 @@ enum NodeAction { /// Command timeout in seconds #[arg(long = "timeout-seconds", default_value_t = 300)] timeout_seconds: i32, + /// Skip the automatic nixos-rebuild test+switch after apply + #[arg(long)] + no_rebuild: bool, }, /// Upload a local image file to a node image cache UploadImage { @@ -1024,6 +1055,12 @@ async fn main() { let info = resolve_node(&cli).unwrap_or_else(|e| fatal(&e)); commands::container::delete(&info, name, *force).await } + Command::Delete { + resource: DeleteResource::DiskLayout { name }, + } => { + let info = resolve_controller(&cli).unwrap_or_else(|e| fatal(&e)); + commands::disk_layout::delete(&info, name).await + } Command::Start { resource: StartResource::Vm { vm_id, target_node }, @@ -1151,6 +1188,12 @@ async fn main() { let info = resolve_controller(&cli).unwrap_or_else(|e| fatal(&e)); commands::conflict::status(&info, *require_healthy).await } + Command::Get { + resource: GetResource::DiskLayouts { target_node }, + } => { + let info = resolve_controller(&cli).unwrap_or_else(|e| fatal(&e)); + commands::disk_layout::list(&info, target_node.as_deref()).await + } Command::Describe { resource: DescribeResource::Vm { name, target_node }, @@ -1188,6 +1231,12 @@ async fn main() { let info = resolve_controller(&cli).unwrap_or_else(|e| fatal(&e)); commands::compliance::report(&info).await } + Command::Describe { + resource: DescribeResource::DiskLayout { name }, + } => { + let info = resolve_controller(&cli).unwrap_or_else(|e| fatal(&e)); + commands::disk_layout::get(&info, name).await + } Command::Node { action: NodeAction::Approve { node_id }, @@ -1286,14 +1335,16 @@ async fn main() { } Command::Node { action: - NodeAction::ApplyDisko { + NodeAction::ApplyDisk { file, apply, timeout_seconds, + no_rebuild, }, } => { let info = resolve_node(&cli).unwrap_or_else(|e| fatal(&e)); - commands::node::apply_disko_layout(&info, file, *apply, *timeout_seconds).await + commands::node::apply_disk_layout(&info, file, *apply, *timeout_seconds, !no_rebuild) + .await } Command::Node { action: @@ -1471,6 +1522,23 @@ async fn main() { } } + Command::Diff { file } => { + let classified = commands::apply::classify_manifest(file) + .unwrap_or_else(|e| fatal(&format!("{e:#}"))); + match classified.kind.as_deref().map(str::to_ascii_lowercase) { + Some(ref k) + if matches!(k.as_str(), "disklayout" | "disk-layout" | "disk_layout") => + { + let info = resolve_controller(&cli).unwrap_or_else(|e| fatal(&e)); + commands::disk_layout::diff_from_file(&info, file).await + } + Some(other) => fatal(&format!( + "kctl diff currently only supports kind: DiskLayout (got {other})" + )), + None => fatal("manifest has no kind: field"), + } + } + Command::Version => { println!("kctl {VERSION}"); Ok(()) diff --git a/crates/kctl/src/output.rs b/crates/kctl/src/output.rs index 65dfb54..4e76162 100644 --- a/crates/kctl/src/output.rs +++ b/crates/kctl/src/output.rs @@ -315,7 +315,7 @@ pub fn print_compliance_report(r: &controller_proto::GetComplianceReportResponse "Infrastructure", &["SOC 2 CC8.1", "PCI DSS 2.2", "PCI DSS 6.3"], ); - field("OS", "NixOS (declarative, atomic updates)"); + field("OS", "kcoreOS (declarative, atomic updates)"); field("Hypervisor", "Cloud Hypervisor (KVM)"); field( "VM Isolation", diff --git a/crates/node-agent/Cargo.toml b/crates/node-agent/Cargo.toml index fc6568d..182bb71 100644 --- a/crates/node-agent/Cargo.toml +++ b/crates/node-agent/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [dependencies] clap = { version = "4", features = ["derive"] } kcore-sanitize = { path = "../kcore-sanitize" } +kcore-disko-types = { path = "../kcore-disko-types" } hostname = "0.4" hyper = { version = "1", features = ["client", "http1"] } hyper-util = { version = "0.1", features = ["tokio", "client-legacy"] } diff --git a/crates/node-agent/src/disk/classifier.rs b/crates/node-agent/src/disk/classifier.rs new file mode 100644 index 0000000..c8381a6 --- /dev/null +++ b/crates/node-agent/src/disk/classifier.rs @@ -0,0 +1,13 @@ +//! Node-agent facade over the shared [`kcore_disko_types`] classifier. +//! +//! The authoritative logic lives in the `kcore-disko-types` crate so the +//! controller pre-flight and the node-agent apply path stay in lockstep. This +//! module simply re-exports the types the `apply_disk_layout_impl` handler +//! uses, so call sites keep working unchanged. + +pub use kcore_disko_types::{ + classify_disk_layout, extract_target_devices, BlockDevice, LsblkSnapshot, Verdict, +}; + +#[allow(unused_imports)] +pub use kcore_disko_types::refusal; diff --git a/crates/node-agent/src/disk/lsblk.rs b/crates/node-agent/src/disk/lsblk.rs new file mode 100644 index 0000000..49e826b --- /dev/null +++ b/crates/node-agent/src/disk/lsblk.rs @@ -0,0 +1,151 @@ +//! lsblk JSON parser feeding [`classifier::LsblkSnapshot`]. +//! +//! We invoke `lsblk -J -p -o NAME,PATH,FSTYPE,MOUNTPOINTS,PKNAME,TYPE` and +//! parse the JSON into a flat [`classifier::LsblkSnapshot`]. Recursion over +//! the `children` tree is explicit so we can propagate `parent_path` without +//! relying on lsblk's own PKNAME when it is empty (which happens for nested +//! `dm-` nodes on some kernels). + +use serde_json::Value; + +use crate::disk::classifier::{BlockDevice, LsblkSnapshot}; + +pub async fn snapshot() -> std::io::Result { + let out = tokio::process::Command::new("lsblk") + .args(["-J", "-p", "-o", "NAME,PATH,FSTYPE,MOUNTPOINTS,PKNAME,TYPE"]) + .output() + .await?; + if !out.status.success() { + return Err(std::io::Error::other(format!( + "lsblk exited with status {}", + out.status + ))); + } + parse(&String::from_utf8_lossy(&out.stdout)) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e)) +} + +pub fn parse(json: &str) -> Result { + let root: Value = serde_json::from_str(json).map_err(|e| format!("parse lsblk json: {e}"))?; + let block_devices = root + .get("blockdevices") + .and_then(Value::as_array) + .ok_or_else(|| "lsblk json missing blockdevices array".to_string())?; + let mut out = Vec::new(); + for dev in block_devices { + walk(dev, None, &mut out); + } + Ok(LsblkSnapshot { devices: out }) +} + +fn walk(node: &Value, parent_path: Option<&str>, out: &mut Vec) { + let path = node + .get("path") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + if path.is_empty() { + return; + } + let kind = node + .get("type") + .and_then(Value::as_str) + .unwrap_or("") + .to_string(); + let fstype = node + .get("fstype") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .map(str::to_string); + let mountpoints = node + .get("mountpoints") + .and_then(Value::as_array) + .map(|arr| { + arr.iter() + .filter_map(Value::as_str) + .filter(|s| !s.is_empty()) + .map(str::to_string) + .collect::>() + }) + .unwrap_or_default(); + let parent = parent_path.map(str::to_string).or_else(|| { + node.get("pkname") + .and_then(Value::as_str) + .filter(|s| !s.is_empty()) + .map(str::to_string) + }); + out.push(BlockDevice { + path: path.clone(), + kind, + fstype, + mountpoints, + parent_path: parent, + }); + if let Some(children) = node.get("children").and_then(Value::as_array) { + for child in children { + walk(child, Some(&path), out); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_flat_disk() { + let json = r#"{ + "blockdevices": [ + {"name":"sda","path":"/dev/sda","fstype":null,"mountpoints":[null],"pkname":null,"type":"disk"} + ] + }"#; + let snap = parse(json).expect("parse"); + assert_eq!(snap.devices.len(), 1); + assert_eq!(snap.devices[0].path, "/dev/sda"); + assert_eq!(snap.devices[0].kind, "disk"); + assert!(snap.devices[0].fstype.is_none()); + assert!(snap.devices[0].mountpoints.is_empty()); + } + + #[test] + fn parses_nested_partition_with_mountpoint() { + let json = r#"{ + "blockdevices": [ + { + "name":"sda","path":"/dev/sda","fstype":null,"mountpoints":[null],"pkname":null,"type":"disk", + "children":[ + {"name":"sda1","path":"/dev/sda1","fstype":"ext4","mountpoints":["/"],"pkname":"/dev/sda","type":"part"} + ] + } + ] + }"#; + let snap = parse(json).expect("parse"); + assert_eq!(snap.devices.len(), 2); + let part = &snap.devices[1]; + assert_eq!(part.path, "/dev/sda1"); + assert_eq!(part.parent_path.as_deref(), Some("/dev/sda")); + assert_eq!(part.fstype.as_deref(), Some("ext4")); + assert_eq!(part.mountpoints, vec!["/".to_string()]); + } + + #[test] + fn propagates_parent_even_when_pkname_missing() { + let json = r#"{ + "blockdevices": [ + { + "name":"nvme0n1","path":"/dev/nvme0n1","fstype":null,"mountpoints":[null],"type":"disk", + "children":[ + {"name":"nvme0n1p1","path":"/dev/nvme0n1p1","fstype":"LVM2_member","mountpoints":[null],"type":"part"} + ] + } + ] + }"#; + let snap = parse(json).expect("parse"); + let part = snap + .devices + .iter() + .find(|d| d.path == "/dev/nvme0n1p1") + .unwrap(); + assert_eq!(part.parent_path.as_deref(), Some("/dev/nvme0n1")); + } +} diff --git a/crates/node-agent/src/disk/mod.rs b/crates/node-agent/src/disk/mod.rs new file mode 100644 index 0000000..05929f9 --- /dev/null +++ b/crates/node-agent/src/disk/mod.rs @@ -0,0 +1,4 @@ +//! DiskLayout node-agent support: classifier + lsblk snapshot helpers. + +pub mod classifier; +pub mod lsblk; diff --git a/crates/node-agent/src/grpc/admin.rs b/crates/node-agent/src/grpc/admin.rs index da087c4..a96cff8 100644 --- a/crates/node-agent/src/grpc/admin.rs +++ b/crates/node-agent/src/grpc/admin.rs @@ -13,6 +13,8 @@ use tracing::{error, info}; use crate::auth::{self, CN_CONTROLLER_PREFIX, CN_KCTL}; use crate::discovery; +use crate::disk::classifier::{self, Verdict}; +use crate::disk::lsblk; use crate::proto; use crate::storage::{self, StorageAdapter}; pub struct AdminService { @@ -26,9 +28,13 @@ const BOOTSTRAP_CERT_DIR: &str = "/etc/kcore/certs"; const INSTALL_LOG_DIR: &str = "/var/log/kcore"; const NIXOS_CONFIG_PATH: &str = "/etc/nixos/configuration.nix"; const IMAGE_CACHE_DIR: &str = "/var/lib/kcore/images"; -const DISKO_MANAGEMENT_MODE_PATH: &str = "/etc/kcore/disko-management-mode"; -const DISKO_MODE_INSTALLER_ONLY: &str = "installer-only"; -const DISKO_MODE_CONTROLLER_MANAGED: &str = "controller-managed"; +const DISK_MANAGEMENT_MODE_PATH: &str = "/etc/kcore/disk-management-mode"; +const DISK_MANAGEMENT_MODE_PATH_LEGACY: &str = "/etc/kcore/disko-management-mode"; +const DISK_MODE_INSTALLER_ONLY: &str = "installer-only"; +const DISK_MODE_CONTROLLER_MANAGED: &str = "controller-managed"; +const DISK_LAYOUT_DIR: &str = "/etc/kcore/disk"; +const DISK_LAYOUT_CURRENT_PATH: &str = "/etc/kcore/disk/current.nix"; +const KCORE_VOLUME_ROOTS: &[&str] = &["/var/lib/kcore/volumes", "/var/lib/kcore/images"]; async fn resolve_nixpkgs_path() -> Option { for candidate in [ @@ -116,20 +122,21 @@ fn validate_disk_path(path: &str, field: &str) -> Result<(), Status> { Ok(()) } -fn normalize_disko_management_mode(raw: &str) -> &'static str { +fn normalize_disk_management_mode(raw: &str) -> &'static str { match raw.trim().to_ascii_lowercase().as_str() { - DISKO_MODE_CONTROLLER_MANAGED => DISKO_MODE_CONTROLLER_MANAGED, - _ => DISKO_MODE_INSTALLER_ONLY, + DISK_MODE_CONTROLLER_MANAGED => DISK_MODE_CONTROLLER_MANAGED, + _ => DISK_MODE_INSTALLER_ONLY, } } -fn read_disko_management_mode() -> &'static str { - let raw = std::fs::read_to_string(DISKO_MANAGEMENT_MODE_PATH) - .unwrap_or_else(|_| DISKO_MODE_INSTALLER_ONLY.to_string()); - normalize_disko_management_mode(&raw) +fn read_disk_management_mode() -> &'static str { + let raw = std::fs::read_to_string(DISK_MANAGEMENT_MODE_PATH) + .or_else(|_| std::fs::read_to_string(DISK_MANAGEMENT_MODE_PATH_LEGACY)) + .unwrap_or_else(|_| DISK_MODE_INSTALLER_ONLY.to_string()); + normalize_disk_management_mode(&raw) } -fn validate_disko_timeout_seconds_or_default(timeout_seconds: i32) -> u64 { +fn validate_disk_timeout_seconds_or_default(timeout_seconds: i32) -> u64 { if timeout_seconds <= 0 { 300 } else { @@ -902,6 +909,193 @@ fn build_install_command_args(req: &proto::InstallToDiskRequest) -> Result Result, Status> { + let mode = read_disk_management_mode(); + + if req.disk_layout_nix.trim().is_empty() { + return Err(Status::invalid_argument("disk_layout_nix cannot be empty")); + } + // The underlying tool is still disko; enforce the disko.devices attribute so + // we fail fast on configuration that cannot possibly be applied. + if !req.disk_layout_nix.contains("disko.devices") { + return Err(Status::invalid_argument( + "disk_layout_nix must define disko.devices", + )); + } + if req.apply && mode != DISK_MODE_CONTROLLER_MANAGED { + return Ok(Response::new(proto::ApplyDiskLayoutResponse { + success: false, + message: "node is in installer-only disk management mode; enable controller-managed mode first" + .to_string(), + mode: mode.to_string(), + refusal_reason: String::new(), + })); + } + + // Safe/dangerous classifier runs only for apply mode. Validate-only calls + // never touch the disks, so they are always permitted. + if req.apply { + let targets = classifier::extract_target_devices(&req.disk_layout_nix); + match lsblk::snapshot().await { + Ok(snap) => { + let verdict = classifier::classify_disk_layout(&targets, &snap, KCORE_VOLUME_ROOTS); + if let Verdict::Dangerous { code, detail } = verdict { + info!( + refusal_reason = code, + detail = %detail, + targets = ?targets, + "refusing dangerous disk layout apply" + ); + return Ok(Response::new(proto::ApplyDiskLayoutResponse { + success: false, + message: detail, + mode: mode.to_string(), + refusal_reason: code.to_string(), + })); + } + } + Err(e) => { + // Fail closed: if we can't confirm the node is idle, refuse. + error!(error = %e, "lsblk snapshot failed; refusing disk layout apply"); + return Ok(Response::new(proto::ApplyDiskLayoutResponse { + success: false, + message: format!("could not inspect block devices to classify request: {e}"), + mode: mode.to_string(), + refusal_reason: "lsblk_probe_failed".to_string(), + })); + } + } + } + + let timeout_seconds = validate_disk_timeout_seconds_or_default(req.timeout_seconds); + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|e| Status::internal(format!("system clock error: {e}")))? + .as_secs(); + + // Always stage the expression in a temp file co-located with the final + // persisted path so the apply-mode rename is atomic (same filesystem). + let staging_dir = if req.apply { + PathBuf::from(DISK_LAYOUT_DIR) + } else { + PathBuf::from("/tmp") + }; + let temp_path = staging_dir.join(format!(".kcore-disk-layout-{timestamp}.nix.tmp")); + + let staging_dir_clone = staging_dir.clone(); + let write_path = temp_path.clone(); + let layout_nix = req.disk_layout_nix.clone(); + tokio::task::spawn_blocking(move || -> std::io::Result<()> { + std::fs::create_dir_all(&staging_dir_clone)?; + std::fs::write(&write_path, layout_nix) + }) + .await + .map_err(|e| Status::internal(format!("task join: {e}")))? + .map_err(|e| { + Status::internal(format!( + "staging disk layout config {}: {e}", + temp_path.display() + )) + })?; + + let run_path = temp_path.clone(); + let mut cmd = Command::new("timeout"); + cmd.args([format!("{timeout_seconds}s")]); + if req.apply { + cmd.arg("disko") + .arg("--mode") + .arg("format,mount") + .arg(run_path.as_os_str()); + } else { + cmd.arg("nix-instantiate") + .arg("--parse") + .arg(run_path.as_os_str()); + } + let output = cmd + .output() + .await + .map_err(|e| Status::internal(format!("running disko command: {e}")))?; + + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + + if !output.status.success() { + // On any failure, drop the staged temp file; we never promote a failed + // apply to the persisted path. + let _ = tokio::fs::remove_file(&temp_path).await; + let detail = if stderr.is_empty() { + format!("disko command failed with status {}", output.status) + } else { + format!("disko command failed: {stderr}") + }; + return Ok(Response::new(proto::ApplyDiskLayoutResponse { + success: false, + message: detail, + mode: mode.to_string(), + refusal_reason: String::new(), + })); + } + + // On successful apply, atomically promote the staged temp file to the + // canonical persisted path so NixOS (via modules/kcore-disko.nix) and the + // reconciler both observe the layout that was actually realised. + let persisted = if req.apply { + let current_path = PathBuf::from(DISK_LAYOUT_CURRENT_PATH); + let staged = temp_path.clone(); + let target = current_path.clone(); + tokio::task::spawn_blocking(move || std::fs::rename(&staged, &target)) + .await + .map_err(|e| Status::internal(format!("task join: {e}")))? + .map_err(|e| { + Status::internal(format!( + "persisting disk layout to {}: {e}", + current_path.display() + )) + })?; + Some(current_path) + } else { + let _ = tokio::fs::remove_file(&temp_path).await; + None + }; + + // Chain nixos-rebuild test -> switch so the persisted layout is evaluated + // by NixOS and activated atomically. We fire-and-forget inside a transient + // systemd unit (same helper ApplyNixConfig uses) to avoid blocking the + // RPC on what is often a multi-minute rebuild. + let mut rebuild_scheduled = false; + if req.apply && req.rebuild { + if let Some(persisted_path) = persisted.clone() { + tokio::spawn(async move { + run_test_then_switch(persisted_path, Vec::new()).await; + }); + rebuild_scheduled = true; + } + } + + let action = if req.apply { "applied" } else { "validated" }; + let base = if stdout.is_empty() { + format!("disk layout {action} successfully") + } else { + format!("disk layout {action} successfully: {stdout}") + }; + let mut detail = base; + if let Some(ref path) = persisted { + detail = format!("{detail}; persisted at {}", path.display()); + } + if rebuild_scheduled { + detail = format!("{detail}; nixos-rebuild test+switch started"); + } + + Ok(Response::new(proto::ApplyDiskLayoutResponse { + success: true, + message: detail, + mode: mode.to_string(), + refusal_reason: String::new(), + })) +} + #[tonic::async_trait] impl proto::node_admin_server::NodeAdmin for AdminService { async fn list_disks( @@ -984,91 +1178,32 @@ impl proto::node_admin_server::NodeAdmin for AdminService { })) } + async fn apply_disk_layout( + &self, + request: Request, + ) -> Result, Status> { + auth::require_peer_insecure_ok(&request, &[CN_KCTL, CN_CONTROLLER_PREFIX])?; + let req = request.into_inner(); + apply_disk_layout_impl(req).await + } + async fn apply_disko_layout( &self, request: Request, ) -> Result, Status> { auth::require_peer_insecure_ok(&request, &[CN_KCTL, CN_CONTROLLER_PREFIX])?; - let req = request.into_inner(); - let mode = read_disko_management_mode(); - - if req.disko_nix.trim().is_empty() { - return Err(Status::invalid_argument("disko_nix cannot be empty")); - } - if !req.disko_nix.contains("disko.devices") { - return Err(Status::invalid_argument( - "disko_nix must define disko.devices", - )); - } - if req.apply && mode != DISKO_MODE_CONTROLLER_MANAGED { - return Ok(Response::new(proto::ApplyDiskoLayoutResponse { - success: false, - message: - "node is in installer-only disko mode; enable controller-managed mode first" - .to_string(), - mode: mode.to_string(), - })); - } - - let timeout_seconds = validate_disko_timeout_seconds_or_default(req.timeout_seconds); - let timestamp = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(|e| Status::internal(format!("system clock error: {e}")))? - .as_secs(); - let temp_path = PathBuf::from(format!("/tmp/kcore-disko-day2-{timestamp}.nix")); - - let write_path = temp_path.clone(); - let disko_nix = req.disko_nix.clone(); - tokio::task::spawn_blocking(move || std::fs::write(&write_path, disko_nix)) - .await - .map_err(|e| Status::internal(format!("task join: {e}")))? - .map_err(|e| { - Status::internal(format!("writing disko config {}: {e}", temp_path.display())) - })?; - - let mut cmd = Command::new("timeout"); - cmd.args([format!("{timeout_seconds}s")]); - if req.apply { - cmd.arg("disko") - .arg("--mode") - .arg("format,mount") - .arg(temp_path.as_os_str()); - } else { - cmd.arg("nix-instantiate") - .arg("--parse") - .arg(temp_path.as_os_str()); - } - let output = cmd - .output() - .await - .map_err(|e| Status::internal(format!("running disko command: {e}")))?; - let _ = tokio::fs::remove_file(&temp_path).await; - - let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - if output.status.success() { - let action = if req.apply { "applied" } else { "validated" }; - let detail = if stdout.is_empty() { - format!("disko layout {action} successfully") - } else { - format!("disko layout {action} successfully: {stdout}") - }; - return Ok(Response::new(proto::ApplyDiskoLayoutResponse { - success: true, - message: detail, - mode: mode.to_string(), - })); - } - - let detail = if stderr.is_empty() { - format!("disko command failed with status {}", output.status) - } else { - format!("disko command failed: {stderr}") + let legacy = request.into_inner(); + let translated = proto::ApplyDiskLayoutRequest { + disk_layout_nix: legacy.disko_nix, + apply: legacy.apply, + timeout_seconds: legacy.timeout_seconds, + rebuild: false, }; + let resp = apply_disk_layout_impl(translated).await?.into_inner(); Ok(Response::new(proto::ApplyDiskoLayoutResponse { - success: false, - message: detail, - mode: mode.to_string(), + success: resp.success, + message: resp.message, + mode: resp.mode, })) } @@ -1981,23 +2116,20 @@ mod tests { } #[test] - fn disko_management_mode_defaults_to_installer_only() { - assert_eq!( - normalize_disko_management_mode(""), - DISKO_MODE_INSTALLER_ONLY - ); + fn disk_management_mode_defaults_to_installer_only() { + assert_eq!(normalize_disk_management_mode(""), DISK_MODE_INSTALLER_ONLY); assert_eq!( - normalize_disko_management_mode("unknown"), - DISKO_MODE_INSTALLER_ONLY + normalize_disk_management_mode("unknown"), + DISK_MODE_INSTALLER_ONLY ); } #[test] - fn disko_timeout_defaults_and_caps() { - assert_eq!(validate_disko_timeout_seconds_or_default(0), 300); - assert_eq!(validate_disko_timeout_seconds_or_default(-1), 300); - assert_eq!(validate_disko_timeout_seconds_or_default(120), 120); - assert_eq!(validate_disko_timeout_seconds_or_default(7200), 3600); + fn disk_timeout_defaults_and_caps() { + assert_eq!(validate_disk_timeout_seconds_or_default(0), 300); + assert_eq!(validate_disk_timeout_seconds_or_default(-1), 300); + assert_eq!(validate_disk_timeout_seconds_or_default(120), 120); + assert_eq!(validate_disk_timeout_seconds_or_default(7200), 3600); } #[test] diff --git a/crates/node-agent/src/issue_screen.rs b/crates/node-agent/src/issue_screen.rs new file mode 100644 index 0000000..275ea91 --- /dev/null +++ b/crates/node-agent/src/issue_screen.rs @@ -0,0 +1,204 @@ +use crate::discovery::{list_disks, list_network_interfaces}; +use crate::proto::{DiskInfo, NetworkInterfaceInfo}; + +const MAX_NICS: usize = 8; +const MAX_DISKS: usize = 8; + +const RESET: &str = "\x1b[0m"; +const BG_DARK: &str = "\x1b[48;5;17m"; +const FG_TITLE: &str = "\x1b[38;5;117m"; +const FG_ACCENT: &str = "\x1b[38;5;45m"; +const FG_TEXT: &str = "\x1b[38;5;252m"; +const FG_MUTED: &str = "\x1b[38;5;110m"; +const FG_WARN: &str = "\x1b[38;5;215m"; + +pub fn should_use_color() -> bool { + std::env::var_os("NO_COLOR").is_none() + && std::env::var("TERM") + .map(|term| term != "dumb") + .unwrap_or(true) +} + +pub fn render_issue(use_color: bool) -> String { + let disks = list_disks(); + let nics = list_network_interfaces(); + + let mut lines = vec![]; + lines.push(style( + use_color, + format!("{BG_DARK}{FG_TITLE} kcoreOS Node Console {RESET}"), + )); + lines.push(style( + use_color, + format!("{FG_ACCENT} Declarative Virtualization Hypervisor {RESET}"), + )); + lines.push(String::new()); + lines.push(style( + use_color, + format!("{FG_TEXT} Username: root Password: kcore {RESET}"), + )); + lines.push(style( + use_color, + format!("{FG_MUTED} Kernel \\r on an \\m (\\l) {RESET}"), + )); + lines.push(String::new()); + + lines.push(style( + use_color, + format!("{FG_ACCENT} Network Interfaces {RESET}"), + )); + lines.push(style( + use_color, + format!( + "{FG_MUTED} {name:<12} {state:<8} {mac:<18} {addr}{RESET}", + name = "NAME", + state = "STATE", + mac = "MAC", + addr = "PRIMARY ADDRESS" + ), + )); + match nics { + Ok(interfaces) => { + let visible = interfaces + .into_iter() + .filter(|iface| iface.name != "lo") + .collect::>(); + render_nics(&mut lines, visible, use_color); + } + Err(err) => lines.push(style( + use_color, + format!("{FG_WARN} NIC discovery unavailable: {err}{RESET}"), + )), + } + lines.push(String::new()); + + lines.push(style(use_color, format!("{FG_ACCENT} Disks {RESET}"))); + lines.push(style( + use_color, + format!( + "{FG_MUTED} {path:<18} {size:<8} {model}{RESET}", + path = "PATH", + size = "SIZE", + model = "MODEL" + ), + )); + match disks { + Ok(devices) => render_disks(&mut lines, devices, use_color), + Err(err) => lines.push(style( + use_color, + format!("{FG_WARN} Disk discovery unavailable: {err}{RESET}"), + )), + } + + lines.push(String::new()); + lines.push(style( + use_color, + format!("{FG_MUTED} Log in to continue configuration and cluster join. {RESET}"), + )); + lines.join("\n") + "\n" +} + +fn render_nics( + lines: &mut Vec, + mut interfaces: Vec, + use_color: bool, +) { + interfaces.sort_by(|a, b| a.name.cmp(&b.name)); + let count = interfaces.len(); + for iface in interfaces.into_iter().take(MAX_NICS) { + let primary = iface + .addresses + .iter() + .find(|addr| !addr.starts_with("fe80")) + .or_else(|| iface.addresses.first()) + .cloned() + .unwrap_or_else(|| "-".to_string()); + lines.push(style( + use_color, + format!( + "{FG_TEXT} {name:<12} {state:<8} {mac:<18} {addr}{RESET}", + name = truncate(&iface.name, 12), + state = truncate(&iface.state, 8), + mac = truncate(&iface.mac_address, 18), + addr = primary + ), + )); + } + if count > MAX_NICS { + lines.push(style( + use_color, + format!( + "{FG_MUTED} ... plus {} additional interface(s){RESET}", + count - MAX_NICS + ), + )); + } + if count == 0 { + lines.push(style( + use_color, + format!("{FG_WARN} No interfaces detected.{RESET}"), + )); + } +} + +fn render_disks(lines: &mut Vec, mut devices: Vec, use_color: bool) { + devices.sort_by(|a, b| a.path.cmp(&b.path)); + let count = devices.len(); + for disk in devices.into_iter().take(MAX_DISKS) { + let model = if disk.model.is_empty() { + "-".to_string() + } else { + disk.model + }; + lines.push(style( + use_color, + format!( + "{FG_TEXT} {path:<18} {size:<8} {model}{RESET}", + path = truncate(&disk.path, 18), + size = truncate(&disk.size, 8), + model = truncate(&model, 42) + ), + )); + } + if count > MAX_DISKS { + lines.push(style( + use_color, + format!( + "{FG_MUTED} ... plus {} additional disk(s){RESET}", + count - MAX_DISKS + ), + )); + } + if count == 0 { + lines.push(style( + use_color, + format!("{FG_WARN} No disks detected.{RESET}"), + )); + } +} + +fn truncate(input: &str, width: usize) -> String { + let chars = input.chars().collect::>(); + if chars.len() <= width { + return input.to_string(); + } + if width <= 3 { + return ".".to_string(); + } + chars[..width - 3].iter().collect::() + "..." +} + +fn style(use_color: bool, value: String) -> String { + if use_color { + value + } else { + value + .replace(BG_DARK, "") + .replace(FG_TITLE, "") + .replace(FG_ACCENT, "") + .replace(FG_TEXT, "") + .replace(FG_MUTED, "") + .replace(FG_WARN, "") + .replace(RESET, "") + } +} diff --git a/crates/node-agent/src/main.rs b/crates/node-agent/src/main.rs index 381a288..191454a 100644 --- a/crates/node-agent/src/main.rs +++ b/crates/node-agent/src/main.rs @@ -5,14 +5,16 @@ mod auth; mod config; mod discovery; +mod disk; mod grpc; +mod issue_screen; mod path_safety; mod registration; mod runtime; mod storage; mod vmm; -use clap::Parser; +use clap::{Args, Parser, Subcommand}; use tokio::signal; use tonic::transport::{Certificate, Identity, Server, ServerTlsConfig}; use tracing::{info, warn}; @@ -62,6 +64,26 @@ struct Cli { /// Allow running without TLS (INSECURE: all RPCs are unauthenticated) #[arg(long)] allow_insecure: bool, + + #[command(subcommand)] + command: Option, +} + +#[derive(Subcommand)] +enum CliCommand { + /// Render an ESX-like pre-login issue screen. + RenderIssue(RenderIssueArgs), +} + +#[derive(Args)] +struct RenderIssueArgs { + /// Destination file for rendered output ("-" for stdout). + #[arg(long, default_value = "/etc/issue")] + output: String, + + /// Disable ANSI colors in output. + #[arg(long)] + no_color: bool, } #[tokio::main] @@ -75,6 +97,18 @@ async fn main() -> anyhow::Result<()> { .init(); let cli = Cli::parse(); + if let Some(CliCommand::RenderIssue(args)) = cli.command { + let use_color = !args.no_color && issue_screen::should_use_color(); + let rendered = issue_screen::render_issue(use_color); + if args.output == "-" { + print!("{rendered}"); + } else { + std::fs::write(&args.output, rendered) + .map_err(|e| anyhow::anyhow!("writing {}: {e}", args.output))?; + } + return Ok(()); + } + let cfg = config::Config::load(&cli.config)?; if cfg.tls.is_none() && !cli.allow_insecure { diff --git a/docs/day-2-filesystem-operations.md b/docs/day-2-filesystem-operations.md index 7837573..d9b4669 100644 --- a/docs/day-2-filesystem-operations.md +++ b/docs/day-2-filesystem-operations.md @@ -19,15 +19,19 @@ This document covers **post-install** operations for storage backends used by kc It focuses on: - ownership and safety controls +- safe/dangerous classifier and refusal semantics - idempotent execution model - operational sequencing - validation and rollback strategies +> Terminology: the operator surface uses the plain word **"disk"** (`DiskLayout`, `kctl node apply-disk`). The underlying partitioning tool is still [disko](https://github.com/nix-community/disko) and is referenced as such in logs, command invocations, and the `kcore.disko.*` NixOS module. + ## Ownership model and safety gates kcore enforces explicit ownership for day-2 disk changes via: -- `/etc/kcore/disko-management-mode` +- `/etc/kcore/disk-management-mode` (new canonical path) +- `/etc/kcore/disko-management-mode` (legacy path — still read as a fallback for one release; `node install` writes the new path and a compatibility symlink) Valid values: @@ -37,31 +41,78 @@ Valid values: ### Rules 1. Install-time always leaves nodes in `installer-only`. -2. `kctl node apply-disko --apply` is rejected in `installer-only`. -3. Validation-only (`kctl node apply-disko -f ...` without `--apply`) remains available for syntax/eval checks. +2. `kctl node apply-disk --apply` is rejected in `installer-only`. +3. Validation-only (`kctl node apply-disk -f ...` without `--apply`) remains available for syntax/eval checks. 4. Promotion to `controller-managed` is a deliberate operator action. +5. Even in `controller-managed` mode, the node-agent runs a **safe/dangerous classifier** against every apply and refuses requests that would touch a block device currently in use (see below). This prevents accidental partitioning operations in normal day-2 workflows. +## Safe / dangerous classifier + +Before running `disko --mode format,mount`, the node-agent snapshots `lsblk -J -p -o NAME,PATH,FSTYPE,MOUNTPOINTS,PKNAME,TYPE` and checks the proposed layout against live state. The request is **refused** when a target device or any descendant is one of: + +- an active kcore volume or image mount (prefixes `/var/lib/kcore/volumes`, `/var/lib/kcore/images`) → `target_device_has_active_kcore_volume` +- an active system mount (`/`, `/boot`, `/boot/efi`, `/nix`, `/nix/store`) → `target_device_has_active_system_mount` +- an active LVM physical volume (`fstype = LVM2_member`) → `target_device_is_active_lvm_pv` +- an active ZFS pool member (`fstype = zfs_member`) → `target_device_is_active_zpool_member` + +When a classifier refuses, `ApplyDiskLayoutResponse.refusal_reason` contains the stable code above and `message` contains the human-readable detail. If lsblk itself cannot be inspected, the agent fails closed with `refusal_reason = "lsblk_probe_failed"`. + +The controller is never expected to drain, stop, migrate, or reboot VMs. The operator quiesces workloads (manually today; via live migration once that lands) before submitting a `DiskLayout`. + +## Recommended workflow: declarative `DiskLayout` resource + +Day-2 disk changes are best driven through the controller as a `kind: DiskLayout` resource instead of direct node pushes. The controller persists the manifest in its replicated DB, classifier-pre-flights it, and the controller-side reconciler dispatches `ApplyDiskLayout` to the owning node and writes the result back into `status`. + +```yaml +kind: DiskLayout +metadata: + name: prod-data-pool +spec: + nodeId: node-prod-01 + layoutNixFile: ./day2-disk.nix # or inline `layoutNix: |` +``` + +```bash +kctl diff -f day2-disk-layout.yaml # controller pre-flight (no writes) +kctl apply -f day2-disk-layout.yaml # creates / updates the DiskLayout +kctl get disk-layouts # list with phase + refusal_reason +kctl describe disk-layout prod-data-pool # full body + status +kctl delete disk-layout prod-data-pool # removes from controller DB; node is untouched +``` + +`kctl diff` calls the controller's read-only `ClassifyDiskLayout` RPC, which extracts the target devices from the Nix body and runs the controller-side pre-flight (structural checks today; live inventory once the replicated block-device table lands). The node-agent classifier is still the authoritative gate on every apply. + +The reconciler retries refused layouts on every tick using the same generation, so the operator drains affected VMs and re-checks `kctl describe disk-layout ` until `phase = applied`. Editing `spec.layoutNix` (or its referenced file) bumps the generation; resubmitting the identical content does not. + +`kctl node apply-disk -f …` remains available for one-off operator pushes and for validation flows where there is no controller (or for nodes still in `installer-only` mode that haven't been registered as DiskLayout targets). + ## RPC and CLI execution path -Day-2 disko operations flow through: +Day-2 disk operations flow through: -1. `kctl` sends `ApplyDiskoLayoutRequest` to `NodeAdmin.ApplyDiskoLayout`. -2. Node-agent validates input contains `disko.devices`. -3. Node-agent executes bounded command via `timeout`: +1. `kctl` sends `ApplyDiskLayoutRequest` to `NodeAdmin.ApplyDiskLayout`. +2. Node-agent validates the Nix text contains `disko.devices`. +3. Node-agent runs the safe/dangerous classifier against live `lsblk`. +4. Node-agent stages the expression under `/etc/kcore/disk/` and executes a bounded command via `timeout`: - validate path: `nix-instantiate --parse ` - apply path: `disko --mode format,mount ` -4. Node-agent returns response containing: +5. On successful apply, node-agent atomically promotes the staged file to `/etc/kcore/disk/current.nix` so NixOS (via `modules/kcore-disko.nix`) and the controller reconciler observe the realised shape. +6. When `rebuild = true` (the default from `kctl`), node-agent chains `nixos-rebuild test` then `nixos-rebuild switch` via a transient systemd unit (`kcore-nix-rebuild.service`), matching the `ApplyNixConfig` flow. +7. Node-agent returns: - `success` - `message` - `mode` (current ownership mode) + - `refusal_reason` (empty on success) Timeout behavior: - `timeout_seconds <= 0` defaults to `300` - timeout is capped server-side (`3600`) +> Back-compat: the `NodeAdmin.ApplyDiskoLayout` RPC and the `kctl node apply-disko` alias still work. They are thin shims that forward to the new RPC/command and will be removed one release later. New manifests and docs should use the `apply-disk` names. + ## Filesystem backend invariants For the `filesystem` backend, current generated layouts use ext4 partitions mounted under: @@ -117,23 +168,24 @@ For `zfs`, disko creates pool; node-agent consumes pool for zvol/dataset operati - backend-specific health (`vgs/pvs/lvs` or `zpool status`) - ensure target disks are correct devices - verify maintenance window and backup posture +- stop / migrate any workload whose backing storage lives on the target devices (the classifier will refuse otherwise) ### 1) Confirm ownership mode ```bash -cat /etc/kcore/disko-management-mode +cat /etc/kcore/disk-management-mode ``` If still `installer-only`, promote intentionally: ```bash -echo controller-managed | sudo tee /etc/kcore/disko-management-mode +echo controller-managed | sudo tee /etc/kcore/disk-management-mode ``` -### 2) Validate disko layout only +### 2) Validate layout only ```bash -kcore-kctl --node node apply-disko -f ./day2-disko.nix +kcore-kctl --node node apply-disk -f ./day2-layout.nix ``` Do not proceed until validation returns success. @@ -141,25 +193,26 @@ Do not proceed until validation returns success. ### 3) Apply with bounded timeout ```bash -kcore-kctl --node node apply-disko \ - -f ./day2-disko.nix \ +kcore-kctl --node node apply-disk \ + -f ./day2-layout.nix \ --apply \ --timeout-seconds 600 ``` -### 4) Reconcile Nix/system view +The node-agent: -Apply node Nix configuration so mount declarations and services are consistent: +- runs the classifier and refuses if any target device is in use; +- persists the applied Nix to `/etc/kcore/disk/current.nix`; +- chains `nixos-rebuild test` → `switch` so mount units and services stay consistent. -```bash -kcore-kctl --node node apply-nix -f ./node-config.nix -``` +Pass `--no-rebuild` only when you are explicitly orchestrating the rebuild yourself (e.g. as part of a controller-driven reconciliation). -### 5) Post-apply verification +### 4) Post-apply verification - `findmnt -R /var/lib/kcore` - verify backend health - verify node-agent active and volume operations still succeed +- `systemctl status kcore-nix-rebuild.service` to confirm the rebuild landed ## Idempotence and repeatability expectations @@ -179,9 +232,15 @@ If a change is not naturally idempotent, treat it as a one-time migration step a - rerun validation - no disk mutation expected +### Classifier refusal + +- `refusal_reason` explains which invariant was violated +- stop / migrate the listed workload and resubmit; never disable the classifier + ### Apply failure - inspect returned stderr in RPC response +- the staged Nix file is **not** promoted to `/etc/kcore/disk/current.nix` when disko fails, so NixOS evaluation is unaffected - verify partial state with `lsblk/findmnt` and backend tools - do not retry blindly; first confirm whether partial partitioning occurred @@ -190,9 +249,14 @@ If a change is not naturally idempotent, treat it as a one-time migration step a - command was bounded; inspect actual system state before re-attempt - increase timeout only when operation is known to be long-running and safe +### Rebuild failure + +- `nixos-rebuild test+switch` runs as `kcore-nix-rebuild.service`; use `journalctl -u kcore-nix-rebuild.service` for logs +- re-applying the same `DiskLayout` is safe (disko is idempotent on matching layout) and re-triggers the rebuild chain + ## Concurrency and maintenance considerations -- avoid concurrent day-2 disko apply operations on the same node +- avoid concurrent day-2 apply operations on the same node - avoid VM placement churn during partitioning operations - for topology-changing operations (VG/pool structural edits), prefer temporary workload quiesce @@ -202,8 +266,7 @@ Minimal additive pattern for `filesystem` backend: 1. Add new disk entry producing `/var/lib/kcore/volumes1`. 2. Validate only. -3. Apply. -4. Reconcile Nix config and verify mount. +3. Apply — classifier sees the new disk is idle, rebuild chain activates the mount. This keeps existing `volumes` mount untouched and reduces blast radius. @@ -212,5 +275,6 @@ This keeps existing `volumes` mount untouched and reduces blast radius. - repartitioning OS/root disk in-place on production nodes - automatic migration of existing volumes between backends - backend conversion (`filesystem -> lvm -> zfs`) without explicit migration tooling +- `--force` overrides of classifier refusals (there are none by design; stop the workload instead) Those are maintenance/reinstall-class workflows and should be handled with dedicated migration procedures. diff --git a/docs/kctl-commands-and-workflows.md b/docs/kctl-commands-and-workflows.md index 119da17..a127ecc 100644 --- a/docs/kctl-commands-and-workflows.md +++ b/docs/kctl-commands-and-workflows.md @@ -317,21 +317,54 @@ Skip rebuild (write only): kctl --node 10.0.0.21:9091 node apply-nix -f ./node-config.nix --no-rebuild ``` -Validate a day-2 disko layout on a node: +Validate a day-2 disk layout on a node (no writes, structural + classifier check only): ```bash -kctl --node 10.0.0.21:9091 node apply-disko -f ./day2-disko.nix +kctl --node 10.0.0.21:9091 node apply-disk -f ./day2-disk.nix ``` -Apply a day-2 disko layout (requires `controller-managed` disko mode on node): +Apply a day-2 disk layout to a single node (requires `controller-managed` +disk-management mode on the node). The node-agent persists the layout to +`/etc/kcore/disk/current.nix` and chains `nixos-rebuild test` then +`nixos-rebuild switch` automatically; pass `--no-rebuild` to skip: ```bash -kctl --node 10.0.0.21:9091 node apply-disko \ - -f ./day2-disko.nix \ +kctl --node 10.0.0.21:9091 node apply-disk \ + -f ./day2-disk.nix \ --apply \ --timeout-seconds 600 ``` +The legacy `node apply-disko` spelling stays as an alias for one release. + +### Declarative DiskLayout via the controller (preferred) + +Submit the same Nix body as a controller-managed `DiskLayout` resource so it +is replicated across controllers and reconciled to the target node: + +```yaml +kind: DiskLayout +metadata: + name: prod-data-pool +spec: + nodeId: node-prod-01 + layoutNixFile: ./day2-disk.nix +``` + +```bash +kctl diff -f day2-disk-layout.yaml +kctl apply -f day2-disk-layout.yaml +kctl get disk-layouts +kctl describe disk-layout prod-data-pool +kctl delete disk-layout prod-data-pool # removes it from the controller; does NOT touch the node +``` + +The controller never touches VMs. If the node-agent's classifier refuses the +layout (the targeted disk currently backs an active kcore volume, an active +LVM PV, or a ZFS pool member), the refusal code surfaces on +`status.refusalReason`. Drain the affected VMs and resubmit the same +manifest — the reconciler retries until the node accepts. + ## 7) Image operations There are two supported VM image flows: @@ -483,7 +516,8 @@ Top-level commands: - `kctl node nics` - `kctl node install --os-disk ... --join-controller ... [--data-disk ...] [--storage-backend filesystem|lvm|zfs] [--disable-vxlan]` - `kctl node apply-nix -f ... [--no-rebuild]` -- `kctl node apply-disko -f ... [--apply] [--timeout-seconds N]` +- `kctl node apply-disk -f ... [--apply] [--timeout-seconds N] [--no-rebuild]` (alias `apply-disko`) +- `kctl apply -f .yaml`, `kctl diff -f .yaml`, `kctl get disk-layouts`, `kctl describe disk-layout `, `kctl delete disk-layout ` - `kctl pull image ` (legacy/manual path) - `kctl node approve ` - `kctl node reject ` @@ -507,7 +541,7 @@ Day-2 operations: 2. review compliance posture with `kctl get compliance-report` (crypto, mTLS, access control, encryption at rest, per-node cert and LUKS status) 3. adjust desired VM running state with `kctl set vm ... --state ...` (or `kctl start/stop vm ...`) 4. update configs with `kctl node apply-nix ...` or `kctl apply ...` -5. for day-2 disk layout changes, use `kctl node apply-disko ...` (validate first, then `--apply`) +5. for day-2 disk layout changes, prefer declarative `kctl apply -f .yaml` (use `kctl diff -f` first); for one-off pushes, `kctl node apply-disk ...` (validate first, then `--apply`) 6. rotate controller cert with `kctl rotate certs --controller ` 7. rotate sub-CA with `kctl rotate sub-ca` diff --git a/docs/storage.md b/docs/storage.md index 356a809..0931c9f 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -111,16 +111,23 @@ For day-2 additions of new data disks, the `disko` CLI is available on installed For kcore, the backing VG or pool is created by disko at install time. VM **volumes** are then created at runtime by the node agent (`lvcreate`, `zfs create -V`, etc.) against the VG or pool named in `node-agent.yaml`. +## Disk management modes (safe installer/controller split) + +> Naming note. The user-facing surface is **disk** (`DiskLayout`, +> `kctl ... disk-layout`, `/etc/kcore/disk-management-mode`). Disko is still +> the underlying tool that performs the partitioning; we just don't lead with +> the brand name on operator-facing surfaces. + ## Disko Ownership Modes (safe installer/controller split) kcore uses an explicit ownership split for disk partitioning state: -- `installer-only` (default): install-time layout is authoritative; controller day-2 disko apply is blocked. -- `controller-managed`: controller/kctl day-2 disko apply is allowed. +- `installer-only` (default): install-time layout is authoritative; controller / `kctl` day-2 disk apply is blocked. +- `controller-managed`: controller / `kctl` day-2 disk apply is allowed. Install-time behavior writes the default mode marker: -- `/etc/kcore/disko-management-mode` contains `installer-only` after `node install`. +- `/etc/kcore/disk-management-mode` contains `installer-only` after `node install` (the legacy `/etc/kcore/disko-management-mode` path is still read as a fallback for one release; a symlink is created for backwards compatibility). This prevents accidental day-2 destructive layout changes on newly installed nodes until operators explicitly promote ownership. @@ -129,25 +136,56 @@ This prevents accidental day-2 destructive layout changes on newly installed nod Operators can promote a node explicitly once runbooks and maintenance windows are in place: ```bash -echo controller-managed | sudo tee /etc/kcore/disko-management-mode +echo controller-managed | sudo tee /etc/kcore/disk-management-mode ``` ### Apply mode safety gate -`kcore-kctl node apply-disko --apply` succeeds only in `controller-managed` mode. -In `installer-only`, the RPC returns a clear rejection and reports the active mode. +`kcore-kctl node apply-disk --apply` (alias `apply-disko` kept for one release) +succeeds only in `controller-managed` mode. In `installer-only`, the RPC +returns a clear rejection and reports the active mode. + +## Day-2 disk workflows + +There are two equivalent ways to drive a day-2 disk change. + +### A) Declarative `kctl apply -f` (recommended) + +Submit a `DiskLayout` manifest to the controller; the controller persists it, +replicates it, and the controller-side reconciler pushes it to the target +node: -## Day-2 Disko workflow (validate first, then apply) +```yaml +kind: DiskLayout +metadata: + name: prod-data-pool +spec: + nodeId: node-prod-01 + layoutNixFile: ./day2-disk.nix # or inline `layoutNix: |` +``` + +```bash +kcore-kctl diff -f day2-disk-layout.yaml # controller pre-flight, no writes +kcore-kctl apply -f day2-disk-layout.yaml # creates/updates DiskLayout +kcore-kctl get disk-layouts # see status (pending / applied / refused) +kcore-kctl describe disk-layout prod-data-pool +``` -### 1) Prepare disko layout file +The node-agent classifier (live `lsblk` based) is the authoritative gate. +If the node-agent refuses (e.g. the targeted disk currently backs an active +kcore volume), the refusal code lands on the layout's `status.refusalReason` +and the operator can drain the affected VMs and resubmit the same manifest — +the generation does not bump on unchanged content, so the reconciler will +retry until the node accepts. -Create a Nix file that defines `disko.devices` for the intended day-2 change (for example adding a new data disk). +### B) Direct node-agent push (`kctl node apply-disk`) -### 2) Validate only (recommended first step) +For one-off operator workflows (validation, manual apply), you can still push +straight to a single node-agent: ```bash -kcore-kctl --node 192.168.40.105:9091 node apply-disko \ - -f ./day2-disko.nix +kcore-kctl --node 192.168.40.105:9091 node apply-disk \ + -f ./day2-disk.nix ``` Validation parses/evaluates layout input without partitioning disks. @@ -155,8 +193,8 @@ Validation parses/evaluates layout input without partitioning disks. ### 3) Apply with bounded timeout (controller-managed mode only) ```bash -kcore-kctl --node 192.168.40.105:9091 node apply-disko \ - -f ./day2-disko.nix \ +kcore-kctl --node 192.168.40.105:9091 node apply-disk \ + -f ./day2-disk.nix \ --apply \ --timeout-seconds 600 ``` @@ -165,10 +203,10 @@ Runtime is bounded server-side (`timeout`), and command output includes success/ ### 4) Reconcile mounts/services -After successful day-2 disko apply, run: - -- `kcore-kctl --node node apply-nix -f ` -- or a full `nixos-rebuild switch` on-node, depending on your operational workflow. +The node-agent persists the applied layout to `/etc/kcore/disk/current.nix` +and chains `nixos-rebuild test` followed by `nixos-rebuild switch` +automatically. There is no longer a separate manual `apply-nix` step — +operators pass `--no-rebuild` only for validation flows or local tests. ## Controller-side disko fragments (multi-node consistency) diff --git a/examples/cluster/prod.yaml b/examples/cluster/prod.yaml new file mode 100644 index 0000000..fd3ff9f --- /dev/null +++ b/examples/cluster/prod.yaml @@ -0,0 +1,5 @@ +kind: Cluster +metadata: + name: prod +spec: + controller: 192.168.40.107:9090 diff --git a/examples/node-install/agent.yaml b/examples/node-install/agent.yaml new file mode 100644 index 0000000..c2690f8 --- /dev/null +++ b/examples/node-install/agent.yaml @@ -0,0 +1,11 @@ +kind: NodeInstall +metadata: + name: kvm-node-151 +spec: + node: 192.168.40.151:9091 + osDisk: /dev/sda + joinControllers: + - 192.168.40.107:9090 + - 192.168.40.105:9090 + dcId: DC1 + insecure: true diff --git a/examples/node-install/controller.yaml b/examples/node-install/controller.yaml new file mode 100644 index 0000000..aeb6557 --- /dev/null +++ b/examples/node-install/controller.yaml @@ -0,0 +1,9 @@ +kind: NodeInstall +metadata: + name: kvm-node-107 +spec: + node: 192.168.40.107:9091 + osDisk: /dev/sda + runController: true + dcId: DC1 + insecure: true diff --git a/examples/node-install/ha-controller.yaml b/examples/node-install/ha-controller.yaml new file mode 100644 index 0000000..0a5c0fa --- /dev/null +++ b/examples/node-install/ha-controller.yaml @@ -0,0 +1,12 @@ +kind: NodeInstall +metadata: + name: kvm-node-105 +spec: + node: 192.168.40.105:9091 + osDisk: /dev/sda + runController: true + joinControllers: + - 192.168.40.107:9090 + storageBackend: lvm + dcId: DC1 + insecure: true diff --git a/flake.nix b/flake.nix index afb4c86..25747dc 100644 --- a/flake.nix +++ b/flake.nix @@ -211,6 +211,7 @@ dashboard = inputs.self.packages.x86_64-linux.kcore-dashboard; diskoPackage = inputs.disko.packages.x86_64-linux.default; kcoreDiskoModule = ./modules/kcore-disko.nix; + kcoreBrandingModule = ./modules/kcore-branding.nix; in { system.stateVersion = "25.05"; @@ -424,12 +425,24 @@ fi echo "======================================================" - echo " KCORE Node - Automated Disk Installer" + echo " kcoreOS Node - Automated Disk Installer" echo "======================================================" echo "" - echo "WARNING: This will ERASE the selected disk and install NixOS!" + echo "WARNING: This will ERASE the selected disk and install kcoreOS!" echo "" + print_install_overview() { + echo "kcoreOS hardware discovery" + echo "--------------------------" + echo "Network interfaces:" + ip -o -4 addr show scope global | awk '{print " - " $2 " " $4}' || true + echo "" + echo "Disks:" + lsblk -d -o NAME,SIZE,MODEL,TYPE | awk '$4 == "disk" {printf " - /dev/%s %s %s\n", $1, $2, $3}' + echo "" + } + print_install_overview + echo "Available disks:" lsblk -d -o NAME,SIZE,TYPE,MODEL | grep disk echo "" @@ -688,7 +701,7 @@ echo "Data disks formatted via disko (backend: $DATA_DISK_MODE)." fi - echo "Generating NixOS hardware configuration..." + echo "Generating kcoreOS hardware configuration..." nixos-generate-config --root /mnt echo "Copying kcore binaries..." @@ -708,6 +721,7 @@ echo "Copying disko configuration..." cp ${kcoreDiskoModule} /mnt/etc/nixos/modules/kcore-disko.nix + cp ${kcoreBrandingModule} /mnt/etc/nixos/modules/kcore-branding.nix cp /tmp/disko-config.nix /mnt/etc/nixos/disko-config.nix echo "Copying kcore config and certificates..." @@ -724,7 +738,9 @@ printf "%s\n" "''${DATA_DISKS[@]}" > /mnt/etc/kcore/data-disks fi # Safe split default: installer owns disk layout until explicitly promoted. - echo "installer-only" > /mnt/etc/kcore/disko-management-mode + echo "installer-only" > /mnt/etc/kcore/disk-management-mode + # Compatibility symlink so node-agents on the previous release still see the mode. + ln -sf disk-management-mode /mnt/etc/kcore/disko-management-mode if [ "$DISABLE_VXLAN" = "true" ]; then touch /mnt/etc/kcore/disable-vxlan @@ -958,7 +974,7 @@ ) fi - # Build LUKS boot config for NixOS + # Build LUKS boot config for kcoreOS ROOT_PART_UUID=$(blkid -s UUID -o value "$ROOT_PART") LUKS_BOOT_CONFIG="" if [ "$LUKS_METHOD" = "tpm2" ]; then @@ -985,7 +1001,7 @@ LUKS_BOOT_CONFIG="''${LUKS_BOOT_CONFIG//ROOT_PART_UUID_PLACEHOLDER/$ROOT_PART_UUID}" fi - # --- Storage-specific NixOS configuration --- + # --- Storage-specific kcoreOS configuration --- STORAGE_BOOT_CONFIG="" HOST_ID=$(echo "$INSTALL_HOSTNAME" | md5sum | head -c 8) case "$DATA_DISK_MODE" in @@ -1005,7 +1021,7 @@ ;; esac - echo "Writing NixOS configuration..." + echo "Writing kcoreOS configuration..." cat > /mnt/etc/nixos/configuration.nix <&2 + rm -f /etc/issue + install -m 0644 /etc/issue.kcore-static /etc/issue + fi + ''; + }; + + systemd.timers.kcore-issue-refresh = { + description = "Periodic kcoreOS issue refresh"; + wantedBy = [ "timers.target" ]; + timerConfig = { + OnBootSec = "20s"; + OnUnitActiveSec = "5min"; + Unit = "kcore-issue-refresh.service"; + }; + }; } diff --git a/modules/kcore-disko.nix b/modules/kcore-disko.nix index ca0a26d..d86ebbf 100644 --- a/modules/kcore-disko.nix +++ b/modules/kcore-disko.nix @@ -128,6 +128,15 @@ let ) cfg.controllerFragments; mergedControllerDevices = foldl' recursiveUpdate { } (map (f: f.devices) sortedFragments); + + # The node-agent atomically writes the most-recent applied DiskLayout to + # cfg.persistedLayoutPath. When the file exists and the node is in + # controller-managed mode, we pull its `disko.devices` attrset into the + # final layout so nixos-rebuild and disko agree on the realised shape. + persistedAvailable = + cfg.managementMode == "controller-managed" && builtins.pathExists cfg.persistedLayoutPath; + persistedImport = if persistedAvailable then import cfg.persistedLayoutPath else { }; + persistedDevices = persistedImport.disko.devices or { }; in { options.kcore.disko = { @@ -171,6 +180,17 @@ in description = "Ownership mode for disko changes."; }; + persistedLayoutPath = mkOption { + type = types.path; + default = /etc/kcore/disk/current.nix; + description = '' + Filesystem path of the most-recent DiskLayout the node-agent applied. + When present and managementMode = "controller-managed", its + `disko.devices` attrset is merged over the declared base layout so + NixOS and disko observe the same realised shape. + ''; + }; + controllerFragments = mkOption { type = types.listOf ( types.submodule { @@ -223,9 +243,13 @@ in ]; } { - disko.devices = recursiveUpdate baseDevices ( - if cfg.managementMode == "controller-managed" then mergedControllerDevices else { } - ); + disko.devices = + let + withFragments = recursiveUpdate baseDevices ( + if cfg.managementMode == "controller-managed" then mergedControllerDevices else { } + ); + in + recursiveUpdate withFragments persistedDevices; } ]); } diff --git a/proto/controller.proto b/proto/controller.proto index 94b39ac..51d3341 100644 --- a/proto/controller.proto +++ b/proto/controller.proto @@ -70,6 +70,21 @@ service Controller { // Volume inventory: list all provisioned VM volumes across the cluster. rpc ListVolumes(ListVolumesRequest) returns (ListVolumesResponse); + + // DiskLayout resource — per-node declarative disk layout. Implemented on the + // node via disko; the operator surface uses the plain word "disk". Each + // DiskLayout targets exactly one node; dangerous requests (those that would + // touch a device currently backing an active workload) are refused by both + // the controller pre-flight and the authoritative node-agent re-classifier. + rpc CreateDiskLayout(CreateDiskLayoutRequest) returns (CreateDiskLayoutResponse); + rpc GetDiskLayout(GetDiskLayoutRequest) returns (GetDiskLayoutResponse); + rpc ListDiskLayouts(ListDiskLayoutsRequest) returns (ListDiskLayoutsResponse); + rpc DeleteDiskLayout(DeleteDiskLayoutRequest) returns (DeleteDiskLayoutResponse); + + // Read-only pre-flight that runs the controller-side classifier without + // persisting or dispatching to the node. Used by `kctl diff disklayout` so + // operators can see SAFE/DANGEROUS before applying. + rpc ClassifyDiskLayout(ClassifyDiskLayoutRequest) returns (ClassifyDiskLayoutResponse); } message RegisterNodeRequest { @@ -976,3 +991,102 @@ message VolumeInfo { string image_format = 7; // raw, qcow2 VmState vm_state = 8; } + +// DiskLayout resource --------------------------------------------------- + +// Lifecycle phase of a DiskLayout. Pending until the reconciler has +// dispatched the node-agent RPC, then Applied / Refused / Failed based on the +// authoritative response. +enum DiskLayoutPhase { + DISK_LAYOUT_PHASE_UNSPECIFIED = 0; + DISK_LAYOUT_PHASE_PENDING = 1; + DISK_LAYOUT_PHASE_APPLIED = 2; + DISK_LAYOUT_PHASE_REFUSED = 3; + DISK_LAYOUT_PHASE_FAILED = 4; +} + +message DiskLayout { + // Stable operator-facing name (metadata.name in the manifest). + string name = 1; + // Target node; every DiskLayout applies to exactly one node. + string node_id = 2; + // Monotonically-increasing generation, bumped on every accepted Create + // (upsert) that produces a content change. + int64 generation = 3; + // Full Nix expression (a `disko.devices` attrset under the hood). + string layout_nix = 4; + google.protobuf.Timestamp created_at = 5; + google.protobuf.Timestamp updated_at = 6; +} + +message DiskLayoutStatus { + int64 observed_generation = 1; + DiskLayoutPhase phase = 2; + // Stable refusal code when phase = REFUSED (mirrors + // ApplyDiskLayoutResponse.refusal_reason). + string refusal_reason = 3; + // Human-readable detail from the last reconcile. + string message = 4; + // Wall-clock of the last observed status transition on the node. + google.protobuf.Timestamp last_transition_at = 5; +} + +message CreateDiskLayoutRequest { + DiskLayout disk_layout = 1; +} + +message CreateDiskLayoutResponse { + bool success = 1; + DiskLayout disk_layout = 2; + ApplyAction action = 3; // upsert outcome: created|updated|unchanged + repeated string changed_fields = 4; // names of mutated fields when action == UPDATED +} + +message GetDiskLayoutRequest { + string name = 1; +} + +message GetDiskLayoutResponse { + DiskLayout disk_layout = 1; + DiskLayoutStatus status = 2; +} + +message ListDiskLayoutsRequest { + // Optional filter: when non-empty, only layouts for this node are returned. + string node_id = 1; +} + +message DiskLayoutSummary { + DiskLayout disk_layout = 1; + DiskLayoutStatus status = 2; +} + +message ListDiskLayoutsResponse { + repeated DiskLayoutSummary disk_layouts = 1; +} + +message DeleteDiskLayoutRequest { + string name = 1; +} + +message DeleteDiskLayoutResponse { + bool success = 1; +} + +message ClassifyDiskLayoutRequest { + // The same payload the operator would send to CreateDiskLayout. + DiskLayout disk_layout = 1; +} + +message ClassifyDiskLayoutResponse { + // When true, the controller pre-flight classifier has no objection. The + // authoritative node-agent classifier will re-check before apply. + bool safe = 1; + // Stable refusal code when safe = false (matches + // ApplyDiskLayoutResponse.refusal_reason). + string refusal_reason = 2; + // Human-readable explanation for the operator. + string detail = 3; + // Target block-device paths the controller extracted from layout_nix. + repeated string target_devices = 4; +} diff --git a/proto/node.proto b/proto/node.proto index f4e628f..5002092 100644 --- a/proto/node.proto +++ b/proto/node.proto @@ -55,9 +55,16 @@ service NodeAdmin { // provided content and (optionally) triggers a nixos-rebuild. rpc ApplyNixConfig(ApplyNixConfigRequest) returns (ApplyNixConfigResponse); - // ApplyDiskoLayout validates/applies a disko layout for day-2 disk operations. - // Apply mode is only allowed when node disko management mode is controller-managed. - rpc ApplyDiskoLayout(ApplyDiskoLayoutRequest) returns (ApplyDiskoLayoutResponse); + // ApplyDiskLayout validates/applies a disk layout (implemented via disko) + // for day-2 disk operations. Apply mode is only allowed when node disk + // management mode is controller-managed. + rpc ApplyDiskLayout(ApplyDiskLayoutRequest) returns (ApplyDiskLayoutResponse); + + // DEPRECATED: forwarding shim for the old name; will be removed one release + // after ApplyDiskLayout has shipped. Use ApplyDiskLayout instead. + rpc ApplyDiskoLayout(ApplyDiskoLayoutRequest) returns (ApplyDiskoLayoutResponse) { + option deprecated = true; + } // EnsureImage verifies/downloads an image file at destination_path and // enforces SHA256 integrity before VM activation. @@ -429,19 +436,47 @@ message ApplyNixConfigResponse { string message = 2; } -message ApplyDiskoLayoutRequest { - // Full Nix expression containing disko.devices definition. - string disko_nix = 1; +message ApplyDiskLayoutRequest { + // Full Nix expression containing the layout (a `disko.devices` attribute set + // under the hood). Disko is still the tool that performs the partitioning; + // the operator surface uses the plain word "disk". + string disk_layout_nix = 1; // When true, run disko --mode format,mount. When false, syntax/eval validation only. bool apply = 2; // Maximum allowed runtime for command execution. int32 timeout_seconds = 3; + // When true (the default) and the layout applies successfully, chain + // nixos-rebuild test followed by nixos-rebuild switch so the persisted + // layout is evaluated by NixOS. When false, skip the rebuild (validation + // or local-only apply flows). + bool rebuild = 4; } -message ApplyDiskoLayoutResponse { +message ApplyDiskLayoutResponse { bool success = 1; string message = 2; string mode = 3; // installer-only or controller-managed + // When the request was refused by the classifier, this is a stable machine + // readable reason. Empty on success or non-classifier failures. + string refusal_reason = 4; +} + +// DEPRECATED: renamed to ApplyDiskLayoutRequest. Keeps `disko_nix` field +// for wire compatibility with clients on the previous release. New clients +// MUST use ApplyDiskLayoutRequest. +message ApplyDiskoLayoutRequest { + option deprecated = true; + string disko_nix = 1; + bool apply = 2; + int32 timeout_seconds = 3; +} + +// DEPRECATED: renamed to ApplyDiskLayoutResponse. +message ApplyDiskoLayoutResponse { + option deprecated = true; + bool success = 1; + string message = 2; + string mode = 3; } message EnsureImageRequest { diff --git a/scripts/check-tla.sh b/scripts/check-tla.sh index c01f1dc..f3dfed6 100644 --- a/scripts/check-tla.sh +++ b/scripts/check-tla.sh @@ -33,6 +33,7 @@ pushd "${SPEC_DIR}" >/dev/null run_tlc "ControllerNodeReconcile.tla" "ControllerNodeReconcile.cfg" run_tlc "ControllerReplication.tla" "ControllerReplication.cfg" run_tlc "CrossDcReplication.tla" "CrossDcReplication.cfg" +run_tlc "DiskLayoutReconcile.tla" "DiskLayoutReconcile.cfg" popd >/dev/null echo "TLA+ model checks passed." diff --git a/specs/tla/DiskLayoutReconcile.cfg b/specs/tla/DiskLayoutReconcile.cfg new file mode 100644 index 0000000..087c703 --- /dev/null +++ b/specs/tla/DiskLayoutReconcile.cfg @@ -0,0 +1,13 @@ +SPECIFICATION Spec +INVARIANTS + TypeOK + Safety_AppliedImpliesSafe + Safety_ObservedNeverAheadOfGeneration + Safety_TerminalPhasesObservedCurrentGen + +CONSTANTS + Resources = {r1, r2} + MaxGen = 3 + +CONSTRAINT + StateConstraint diff --git a/specs/tla/DiskLayoutReconcile.tla b/specs/tla/DiskLayoutReconcile.tla new file mode 100644 index 0000000..400df6e --- /dev/null +++ b/specs/tla/DiskLayoutReconcile.tla @@ -0,0 +1,160 @@ +------------------------------- MODULE DiskLayoutReconcile ------------------------------- +\* Per-resource state machine for the controller-orchestrated DiskLayout resource. +\* +\* Models the reconciliation loop between the controller (which owns +\* `DiskLayoutRow.generation`) and the node-agent (which owns +\* `DiskLayoutStatus.observed_generation` + `phase`). The central safety +\* claim is that no resource ever transitions to `Applied` unless the +\* node-agent's authoritative classifier has just declared the proposed +\* layout safe against live `lsblk` state. +\* +\* This spec deliberately abstracts over +\* * the actual Nix evaluator and disko invocation (collapsed into a +\* boolean `Safe[r]`), +\* * network failures (the reconciler simply retries every tick), +\* * controller HA / replication (covered by ControllerReplication.tla). +\* +\* See `crates/controller/src/disk_reconciler.rs` for the production loop +\* and `crates/kcore-disko-types/src/lib.rs` for the classifier. + +EXTENDS Naturals, TLC + +CONSTANTS + Resources, \* finite set of DiskLayout names + MaxGen \* upper bound on the controller-side `generation` + +ASSUME /\ Resources # {} + /\ MaxGen \in Nat + /\ MaxGen >= 1 + +Phases == {"Pending", "Applied", "Refused", "Failed"} + +VARIABLES + Generation, \* [r |-> Nat] controller-owned desired generation + Observed, \* [r |-> Nat] node-observed generation + Phase, \* [r |-> Phases] last phase reported by node-agent + Safe \* [r |-> BOOLEAN] classifier verdict for current generation + +Vars == <> + +\* ----- Initial state ----------------------------------------------------------- +Init == + /\ Generation = [r \in Resources |-> 1] + /\ Observed = [r \in Resources |-> 0] + /\ Phase = [r \in Resources |-> "Pending"] + /\ Safe \in [Resources -> BOOLEAN] + +\* ----- Operator actions -------------------------------------------------------- + +\* Operator submits a *new* layout body for `r`. We bump Generation, +\* reset Observed to "behind" (so the reconciler must pick it up) and +\* clear the phase to Pending. The classifier verdict for the *new* +\* body is freshly chosen (the old one is stale). +SubmitNewLayout(r) == + /\ Generation[r] < MaxGen + /\ Generation' = [Generation EXCEPT ![r] = @ + 1] + /\ Observed' = [Observed EXCEPT ![r] = 0] + /\ Phase' = [Phase EXCEPT ![r] = "Pending"] + /\ \E b \in BOOLEAN : Safe' = [Safe EXCEPT ![r] = b] + +\* Operator drains affected VMs / clears blockers, flipping the +\* node-side classifier verdict from dangerous to safe. This is the +\* only transition that may turn Safe[r] from FALSE to TRUE without +\* a Generation bump. +ClearBlockers(r) == + /\ ~Safe[r] + /\ Safe' = [Safe EXCEPT ![r] = TRUE] + /\ UNCHANGED <> + +\* ----- Reconciler tick (controller-driven) ------------------------------------- + +\* The reconciler picks a layout that is behind, calls the node-agent, +\* and writes back observed_generation + phase based on the verdict. +\* +\* Three terminal-for-this-generation outcomes are modeled: +\* * Safe -> Applied +\* * !Safe -> Refused (operator must clear blockers and the next +\* reconcile will retry the same generation) +\* * Safe -> Failed (disko ran but nixos-rebuild bombed; modeled +\* as a non-deterministic infrastructure failure) +ReconcileApplied(r) == + /\ Observed[r] < Generation[r] + /\ Safe[r] + /\ Phase' = [Phase EXCEPT ![r] = "Applied"] + /\ Observed' = [Observed EXCEPT ![r] = Generation[r]] + /\ UNCHANGED <> + +ReconcileRefused(r) == + /\ Observed[r] < Generation[r] + /\ ~Safe[r] + /\ Phase' = [Phase EXCEPT ![r] = "Refused"] + \* Observed stays behind: the same generation will be retried. + /\ UNCHANGED <> + +ReconcileFailed(r) == + /\ Observed[r] < Generation[r] + /\ Safe[r] + /\ Phase' = [Phase EXCEPT ![r] = "Failed"] + /\ Observed' = [Observed EXCEPT ![r] = Generation[r]] + /\ UNCHANGED <> + +Noop == UNCHANGED Vars + +Next == + \/ \E r \in Resources : + \/ SubmitNewLayout(r) + \/ ClearBlockers(r) + \/ ReconcileApplied(r) + \/ ReconcileRefused(r) + \/ ReconcileFailed(r) + \/ Noop + +FairnessApplied == \A r1 \in Resources : WF_Vars(ReconcileApplied(r1)) +FairnessRefused == \A r2 \in Resources : WF_Vars(ReconcileRefused(r2)) + +Spec == + Init /\ [][Next]_Vars /\ FairnessApplied /\ FairnessRefused + +\* ----- Type & state-space bound ---------------------------------------------- + +TypeOK == + /\ Generation \in [Resources -> 1..MaxGen] + /\ Observed \in [Resources -> 0..MaxGen] + /\ Phase \in [Resources -> Phases] + /\ Safe \in [Resources -> BOOLEAN] + +StateConstraint == + \A r \in Resources : + /\ Generation[r] <= MaxGen + /\ Observed[r] <= Generation[r] + +\* ----- Safety invariants ------------------------------------------------------ + +\* The headline guarantee. Any time we are at phase Applied for `r`, +\* the most recent reconcile saw Safe[r] = TRUE. Equivalently: +\* "no Applied without classifier approval". +Safety_AppliedImpliesSafe == + \A r \in Resources : + Phase[r] = "Applied" => Safe[r] + +\* The reconciler can only catch up; it never moves Observed past the +\* controller's desired generation. +Safety_ObservedNeverAheadOfGeneration == + \A r \in Resources : Observed[r] <= Generation[r] + +\* When Phase records a node-acknowledged outcome (Applied or Failed), +\* Observed must equal Generation. Refused / Pending may legitimately +\* leave Observed behind so the reconciler retries. +Safety_TerminalPhasesObservedCurrentGen == + \A r \in Resources : + Phase[r] \in {"Applied", "Failed"} => Observed[r] = Generation[r] + +\* ----- Liveness -------------------------------------------------------------- + +\* If a layout is safe and the operator stops submitting new +\* generations, the reconciler eventually reaches Applied. +Liveness_SafeEventuallyApplied == + \A r \in Resources : + [](Safe[r] /\ Observed[r] < Generation[r] => <>(Phase[r] = "Applied")) + +============================================================================================== diff --git a/specs/tla/README.md b/specs/tla/README.md index 0dc0d32..5f6a196 100644 --- a/specs/tla/README.md +++ b/specs/tla/README.md @@ -23,6 +23,16 @@ The intent is to check protocol-level safety/liveness properties with TLC. - Separates intra-DC and cross-DC anti-entropy actions. - Checks no-double-apply plus eventual cross-DC convergence after work is applied. +- `DiskLayoutReconcile.tla` + `DiskLayoutReconcile.cfg` + - Per-resource state machine for the controller-orchestrated `DiskLayout`. + - Models `Generation` (controller-owned), `Observed` + `Phase` (node-owned), + and the node-agent classifier verdict `Safe`. + - Central safety invariant `Safety_AppliedImpliesSafe`: no resource ever + enters phase `Applied` without the node-agent classifier currently + declaring the proposed layout safe. + - Auxiliary invariants check that `Observed <= Generation` and that + terminal phases (`Applied`/`Failed`) imply the reconciler caught up. + ## Running locally Preferred: diff --git a/tools/tla2tools.jar b/tools/tla2tools.jar new file mode 100644 index 0000000..fed0509 Binary files /dev/null and b/tools/tla2tools.jar differ