diagnostics: LPG residency accounting + compact/lazy experiments

crates/grafeo-common/src/memory/mod.rs

@@ -27,4 +27,7 @@ pub use buffer::{
 pub use bump::BumpAllocator;
 pub use pool::ObjectPool;
 pub use reporter::MemoryReporter;
-pub use usage::{IndexMemory, MvccMemory, NamedMemory, StoreMemory, StringPoolMemory};
+pub use usage::{
+    AdjacencyCapacityMemory, IndexMemory, LpgResidencyMemory, MvccMemory, NamedMemory,
+    PropertyColumnMemory, PropertyStorageMemory, StoreMemory, StringPoolMemory,
+};

crates/grafeo-common/src/memory/usage.rs

@@ -20,6 +20,30 @@ pub struct StoreMemory {
     pub edge_properties_bytes: usize,
     /// Number of property columns (node + edge).
     pub property_column_count: usize,
+    /// Hash-map slot bytes for node property columns (capacity × entry).
+    #[serde(default)]
+    pub node_property_map_slot_bytes: usize,
+    /// Decoded `Value` payload bytes inside node property columns (strings/lists/…).
+    #[serde(default)]
+    pub node_property_decoded_payload_bytes: usize,
+    /// String/bytes payload subset of node property decoded values.
+    #[serde(default)]
+    pub node_property_string_payload_bytes: usize,
+    /// Unused map capacity waste in node property columns.
+    #[serde(default)]
+    pub node_property_capacity_waste_bytes: usize,
+    /// Hash-map slot bytes for edge property columns.
+    #[serde(default)]
+    pub edge_property_map_slot_bytes: usize,
+    /// Decoded `Value` payload bytes inside edge property columns.
+    #[serde(default)]
+    pub edge_property_decoded_payload_bytes: usize,
+    /// String/bytes payload subset of edge property decoded values.
+    #[serde(default)]
+    pub edge_property_string_payload_bytes: usize,
+    /// Unused map capacity waste in edge property columns.
+    #[serde(default)]
+    pub edge_property_capacity_waste_bytes: usize,
 }
 
 impl StoreMemory {
@@ -32,6 +56,139 @@ impl StoreMemory {
     }
 }
 
+/// Per-column residency attribution for one property key.
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct PropertyColumnMemory {
+    /// Property key name.
+    pub key: String,
+    /// Live entries in the hot map.
+    pub entry_count: usize,
+    /// Hot map capacity (buckets reserved).
+    pub map_capacity: usize,
+    /// `capacity × (Id + Value + 1)` slot estimate.
+    pub map_slot_bytes: usize,
+    /// Sum of [`crate::types::Value::estimated_size_bytes`] over hot values.
+    pub decoded_payload_bytes: usize,
+    /// String + bytes payload subset of decoded values.
+    pub string_payload_bytes: usize,
+    /// Compressed column backing (0 when uncompressed).
+    pub compressed_bytes: usize,
+    /// `(capacity − len) × entry` waste in the hot map.
+    pub capacity_waste_bytes: usize,
+    /// Values held only in compressed form (not in the hot map).
+    pub compressed_entry_count: usize,
+}
+
+impl PropertyColumnMemory {
+    /// Estimated total for this column (slots + payloads + compressed).
+    #[must_use]
+    pub fn total_bytes(&self) -> usize {
+        self.map_slot_bytes + self.decoded_payload_bytes + self.compressed_bytes
+    }
+}
+
+/// Aggregated property-storage residency (node or edge side).
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct PropertyStorageMemory {
+    /// Outer column-map overhead.
+    pub map_overhead_bytes: usize,
+    /// Per-column breakdown (sorted by `total_bytes` descending when produced).
+    pub columns: Vec<PropertyColumnMemory>,
+    /// Sum of column map-slot bytes.
+    pub total_map_slot_bytes: usize,
+    /// Sum of decoded payloads.
+    pub total_decoded_payload_bytes: usize,
+    /// Sum of string/bytes payloads.
+    pub total_string_payload_bytes: usize,
+    /// Sum of compressed backings.
+    pub total_compressed_bytes: usize,
+    /// Sum of hot-map capacity waste.
+    pub total_capacity_waste_bytes: usize,
+    /// `map_overhead + Σ column totals`.
+    pub total_bytes: usize,
+}
+
+impl PropertyStorageMemory {
+    /// Recomputes aggregate totals from columns + outer map overhead.
+    pub fn compute_total(&mut self) {
+        self.total_map_slot_bytes = self.columns.iter().map(|c| c.map_slot_bytes).sum();
+        self.total_decoded_payload_bytes =
+            self.columns.iter().map(|c| c.decoded_payload_bytes).sum();
+        self.total_string_payload_bytes =
+            self.columns.iter().map(|c| c.string_payload_bytes).sum();
+        self.total_compressed_bytes = self.columns.iter().map(|c| c.compressed_bytes).sum();
+        self.total_capacity_waste_bytes =
+            self.columns.iter().map(|c| c.capacity_waste_bytes).sum();
+        self.total_bytes = self.map_overhead_bytes
+            + self.total_map_slot_bytes
+            + self.total_decoded_payload_bytes
+            + self.total_compressed_bytes;
+    }
+}
+
+/// Adjacency list capacity vs used-byte attribution.
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct AdjacencyCapacityMemory {
+    /// Nodes with an adjacency list.
+    pub node_count: usize,
+    /// Outer list-map capacity.
+    pub list_map_capacity: usize,
+    /// Outer list-map overhead bytes.
+    pub list_map_overhead_bytes: usize,
+    /// Bytes occupied by live hot entries (destinations + edge ids used len).
+    pub hot_used_bytes: usize,
+    /// Bytes reserved by hot chunk Vec capacities.
+    pub hot_capacity_bytes: usize,
+    /// Compressed cold-chunk bytes.
+    pub cold_bytes: usize,
+    /// Delta / deleted / skip-index reserved bytes.
+    pub aux_capacity_bytes: usize,
+    /// `hot_capacity − hot_used` (+ list map slack approximated separately).
+    pub capacity_waste_bytes: usize,
+    /// Total estimated heap (`list_map_overhead + hot_capacity + cold + aux`).
+    pub total_bytes: usize,
+}
+
+impl AdjacencyCapacityMemory {
+    /// Recomputes `capacity_waste_bytes` and `total_bytes`.
+    pub fn compute_total(&mut self) {
+        self.capacity_waste_bytes = self.hot_capacity_bytes.saturating_sub(self.hot_used_bytes);
+        self.total_bytes = self.list_map_overhead_bytes
+            + self.hot_capacity_bytes
+            + self.cold_bytes
+            + self.aux_capacity_bytes;
+    }
+}
+
+/// Full LPG residency attribution (properties + adjacency capacities).
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct LpgResidencyMemory {
+    /// Node property columns.
+    pub node_properties: PropertyStorageMemory,
+    /// Edge property columns.
+    pub edge_properties: PropertyStorageMemory,
+    /// Forward adjacency capacity detail.
+    pub forward_adjacency: AdjacencyCapacityMemory,
+    /// Backward adjacency capacity detail (empty when disabled).
+    pub backward_adjacency: AdjacencyCapacityMemory,
+    /// Sum of property + adjacency totals.
+    pub total_bytes: usize,
+}
+
+impl LpgResidencyMemory {
+    /// Recomputes `total_bytes` from children.
+    pub fn compute_total(&mut self) {
+        self.node_properties.compute_total();
+        self.edge_properties.compute_total();
+        self.forward_adjacency.compute_total();
+        self.backward_adjacency.compute_total();
+        self.total_bytes = self.node_properties.total_bytes
+            + self.edge_properties.total_bytes
+            + self.forward_adjacency.total_bytes
+            + self.backward_adjacency.total_bytes;
+    }
+}
+
 /// Memory used by index structures.
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct IndexMemory {
@@ -41,6 +198,12 @@ pub struct IndexMemory {
     pub forward_adjacency_bytes: usize,
     /// Backward adjacency lists (0 if disabled).
     pub backward_adjacency_bytes: usize,
+    /// Forward adjacency capacity waste (`capacity − used` on hot chunks).
+    #[serde(default)]
+    pub forward_adjacency_capacity_waste_bytes: usize,
+    /// Backward adjacency capacity waste.
+    #[serde(default)]
+    pub backward_adjacency_capacity_waste_bytes: usize,
     /// Label index (label_id -> node set).
     pub label_index_bytes: usize,
     /// Node-to-labels reverse index.

crates/grafeo-common/src/types/value.rs

@@ -425,6 +425,28 @@ impl Value {
             }
         }
     }
+
+    /// String / bytes payload bytes only (subset of [`Self::estimated_size_bytes`]).
+    ///
+    /// Used by LPG residency accounting to attribute ArcStr/bytes separately
+    /// from map-slot and compressed-column bytes. Nested lists/maps recurse.
+    #[must_use]
+    pub fn string_payload_bytes(&self) -> usize {
+        match self {
+            Value::String(s) => s.len(),
+            Value::Bytes(b) => b.len(),
+            Value::List(items) => items.iter().map(Value::string_payload_bytes).sum(),
+            Value::Map(m) => m
+                .iter()
+                .map(|(k, v)| k.as_ref().len() + v.string_payload_bytes())
+                .sum(),
+            Value::Path { nodes, edges } => {
+                nodes.iter().map(Value::string_payload_bytes).sum::<usize>()
+                    + edges.iter().map(Value::string_payload_bytes).sum::<usize>()
+            }
+            _ => 0,
+        }
+    }
 }
 
 impl fmt::Debug for Value {