From 0d1211abf4e7bef924528b773e2943d428fdc16d Mon Sep 17 00:00:00 2001 From: jarmen423 Date: Thu, 30 Jul 2026 19:04:48 +0000 Subject: [PATCH 1/2] fix(storage): record truthful per-section directory versions (G-F0.1) Preserve each Section::version() in the container directory via write_versioned_sections and engine flush. Keep legacy write_sections at directory version 1, skip unknown optional section types, fail closed on unknown required types, and document historical outer-v1 compatibility. --- crates/grafeo-engine/src/database/flush.rs | 27 +- .../tests/section_directory_versions.rs | 261 ++++++++++++++++++ .../grafeo-storage/src/container/directory.rs | 78 +++++- crates/grafeo-storage/src/file/manager.rs | 207 +++++++++++++- docs/architecture/storage/container-format.md | 73 +++-- 5 files changed, 599 insertions(+), 47 deletions(-) create mode 100644 crates/grafeo-engine/tests/section_directory_versions.rs diff --git a/crates/grafeo-engine/src/database/flush.rs b/crates/grafeo-engine/src/database/flush.rs index 9732da771..556ecf4bb 100644 --- a/crates/grafeo-engine/src/database/flush.rs +++ b/crates/grafeo-engine/src/database/flush.rs @@ -62,11 +62,17 @@ pub(super) fn flush( maybe_crash("flush:before_serialize"); // Collect sections to write based on flush reason - // Write all sections (dirty or not for Explicit, only dirty for Checkpoint) - let mut targets: Vec<(SectionType, Vec)> = Vec::new(); + // Write all sections (dirty or not for Explicit, only dirty for Checkpoint). + // Retain each section's declared format version for the directory entry + // (G-F0.1); do not hard-code version 1 at the shared writer. + let mut targets: Vec<(SectionType, u8, Vec)> = Vec::new(); for section in sections { if reason == FlushReason::Explicit || section.is_dirty() { - targets.push((section.section_type(), section.serialize()?)); + targets.push(( + section.section_type(), + section.version(), + section.serialize()?, + )); } } // If nothing is dirty on a periodic checkpoint, skip the write entirely. @@ -81,11 +87,13 @@ pub(super) fn flush( maybe_crash("flush:after_serialize"); - // Write sections to container - let section_refs: Vec<(SectionType, &[u8])> = - targets.iter().map(|(t, d)| (*t, d.as_slice())).collect(); + // Write sections to container with truthful per-section directory versions. + let section_refs: Vec<(SectionType, u8, &[u8])> = targets + .iter() + .map(|(t, v, d)| (*t, *v, d.as_slice())) + .collect(); - fm.write_sections( + fm.write_versioned_sections( §ion_refs, context.epoch, context.transaction_id, @@ -95,7 +103,10 @@ pub(super) fn flush( // Mark all written sections as clean for section in sections { - if targets.iter().any(|(t, _)| *t == section.section_type()) { + if targets + .iter() + .any(|(t, _, _)| *t == section.section_type()) + { section.mark_clean(); } } diff --git a/crates/grafeo-engine/tests/section_directory_versions.rs b/crates/grafeo-engine/tests/section_directory_versions.rs new file mode 100644 index 000000000..fb4a6c386 --- /dev/null +++ b/crates/grafeo-engine/tests/section_directory_versions.rs @@ -0,0 +1,261 @@ +//! G-F0.1: truthful per-section directory versions through engine flush. +//! +//! New checkpoints must record each section's declared `Section::version()` +//! in the container directory. Historical outer-v1 entries with supported +//! payloads must remain readable. Unsupported CompactStore payload versions +//! must fail closed (no silent misparse). + +#![cfg(all(feature = "grafeo-file", feature = "lpg"))] + +use grafeo_common::storage::{Section, SectionType}; +use grafeo_common::types::Value; +use grafeo_engine::{Config, GrafeoDB}; +use grafeo_storage::file::GrafeoFileManager; + +fn directory_version(path: &std::path::Path, section_type: SectionType) -> u8 { + let manager = GrafeoFileManager::open_read_only(path).expect("open container"); + let dir = manager + .read_section_directory() + .expect("read directory") + .expect("v2 directory present"); + let entry = dir + .find(section_type) + .unwrap_or_else(|| panic!("missing section {section_type:?}")); + let version = entry.version; + manager.close().ok(); + version +} + +#[test] +fn checkpoint_records_catalog_and_lpg_declared_versions() { + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("catalog_lpg.grafeo"); + + { + let db = GrafeoDB::with_config(Config::persistent(&path)).unwrap(); + let session = db.session(); + session + .execute("INSERT (:Person {name: 'Ada', age: 36})") + .unwrap(); + db.wal_checkpoint().unwrap(); + db.close().unwrap(); + } + + // CatalogSection and LpgStoreSection both declare version 2 at this pin. + assert_eq!(directory_version(&path, SectionType::Catalog), 2); + assert_eq!(directory_version(&path, SectionType::LpgStore), 2); + + // Reopen proves historical-style outer metadata did not break recovery. + let db = GrafeoDB::with_config(Config::persistent(&path)).unwrap(); + assert_eq!(db.node_count(), 1); + db.close().unwrap(); +} + +#[test] +#[cfg(feature = "compact-store")] +fn compact_checkpoint_records_compact_store_declared_version() { + use grafeo_core::graph::compact::section::CompactStoreSection; + + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("compact_versions.grafeo"); + + { + let mut db = GrafeoDB::with_config(Config::persistent(&path)).unwrap(); + db.execute("INSERT (:Doc {title: 't', body: 'hello'})") + .unwrap(); + db.compact().unwrap(); + assert!( + db.layered_store().is_some(), + "layered CompactStore must be installed after compact()" + ); + db.close().unwrap(); + } + + let declared = CompactStoreSection::empty().version(); + assert_eq!( + directory_version(&path, SectionType::CompactStore), + declared, + "directory version must match CompactStoreSection::version()" + ); + // Layered compact path also emits overlay LPG. + assert_eq!(directory_version(&path, SectionType::LpgStore), 2); + + // Reopen through the layered path (node_count alone can under-count base). + let db = GrafeoDB::with_config(Config::persistent(&path)).unwrap(); + assert!( + db.layered_store().is_some(), + "reopen must reconstruct LayeredStore from CompactStore section" + ); + let result = db + .session() + .execute("MATCH (d:Doc) RETURN count(d)") + .unwrap(); + assert_eq!(result.rows()[0][0], Value::Int64(1)); + db.close().unwrap(); +} + +#[test] +#[cfg(feature = "compact-store")] +fn historical_outer_v1_compact_payload_still_opens() { + // Simulate a pre-F0.1 writer: CompactStore payload is current, but the + // directory entry still claims version 1. + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("outer_v1_compact.grafeo"); + + { + let mut db = GrafeoDB::with_config(Config::persistent(&path)).unwrap(); + db.execute("INSERT (:Doc {title: 'keep-me'})").unwrap(); + db.compact().unwrap(); + db.close().unwrap(); + } + + // Rewrite directory with outer version 1 while preserving payload bytes. + { + let manager = GrafeoFileManager::open(&path).unwrap(); + let section_dir = manager + .read_section_directory() + .unwrap() + .expect("directory"); + let mut rewritten: Vec<(SectionType, Vec)> = Vec::new(); + for entry in section_dir.entries() { + let data = manager.read_section_data(entry).unwrap(); + rewritten.push((entry.section_type, data)); + } + let refs: Vec<(SectionType, &[u8])> = rewritten + .iter() + .map(|(t, d)| (*t, d.as_slice())) + .collect(); + // write_sections always records directory version 1 (legacy path). + manager + .write_sections(&refs, 1, 1, 1, 0) + .expect("rewrite with outer v1"); + manager.close().unwrap(); + } + + assert_eq!(directory_version(&path, SectionType::CompactStore), 1); + + let db = GrafeoDB::with_config(Config::persistent(&path)).unwrap(); + assert!( + db.layered_store().is_some(), + "outer-v1 CompactStore entry must still reconstruct LayeredStore" + ); + let result = db + .session() + .execute("MATCH (d:Doc) RETURN d.title") + .unwrap(); + let titles: Vec = result + .rows() + .iter() + .filter_map(|r| match &r[0] { + Value::String(s) => Some(s.to_string()), + _ => None, + }) + .collect(); + assert_eq!(titles, vec!["keep-me".to_string()]); + db.close().unwrap(); +} + +#[test] +#[cfg(feature = "compact-store")] +fn compact_store_rejects_unsupported_future_payload_version() { + // Older reader (this pin understands CompactStore ≤3) must fail closed + // on a future payload version rather than silently misparsing. + use grafeo_core::graph::compact::section::CompactStoreSection; + + // Minimal GCST header: magic + version 4 + zero flags, CRC over header. + let mut payload = Vec::new(); + payload.extend_from_slice(b"GCST"); + payload.push(4); // unsupported future payload version + payload.push(0); // flags + let crc = crc32fast::hash(&payload); + payload.extend_from_slice(&crc.to_le_bytes()); + + let mut section = CompactStoreSection::empty(); + let err = section + .deserialize(&payload) + .expect_err("future CompactStore payload must fail closed"); + let msg = err.to_string(); + assert!( + msg.contains("unsupported CompactStore") || msg.contains("version 4"), + "expected unsupported-version error, got: {msg}" + ); +} + +#[test] +#[cfg(feature = "vector-index")] +fn checkpoint_records_vector_store_declared_version() { + use grafeo_core::index::vector::VectorStoreSection; + + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("vector_versions.grafeo"); + + { + let db = GrafeoDB::with_config(Config::persistent(&path)).unwrap(); + let session = db.session(); + session + .execute("INSERT (:Doc {emb: [0.1, 0.2, 0.3]})") + .unwrap(); + db.create_vector_index("Doc", "emb", Some(3), Some("cosine"), None, None, None) + .unwrap(); + db.wal_checkpoint().unwrap(); + db.close().unwrap(); + } + + let declared = VectorStoreSection::new(Vec::new()).version(); + assert_eq!( + directory_version(&path, SectionType::VectorStore), + declared + ); +} + +#[test] +#[cfg(feature = "text-index")] +fn checkpoint_records_text_index_declared_version() { + use grafeo_core::index::text::TextIndexSection; + + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("text_versions.grafeo"); + + { + let db = GrafeoDB::with_config(Config::persistent(&path)).unwrap(); + let session = db.session(); + session + .execute("INSERT (:Article {title: 'hello world'})") + .unwrap(); + db.create_text_index("Article", "title").unwrap(); + db.wal_checkpoint().unwrap(); + db.close().unwrap(); + } + + let declared = TextIndexSection::new(Vec::new()).version(); + assert_eq!(directory_version(&path, SectionType::TextIndex), declared); +} + +#[test] +fn property_index_directory_version_via_shared_writer() { + // Property indexes are not emitted as a standalone container section by + // the current layered/LPG build_sections path; the shared writer still + // must preserve a declared PropertyIndex directory version when asked. + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("property_index_version.grafeo"); + + let manager = GrafeoFileManager::create(&path).unwrap(); + manager + .write_versioned_sections( + &[(SectionType::PropertyIndex, 1, b"prop-index-payload".as_slice())], + 1, + 1, + 0, + 0, + ) + .unwrap(); + let section_dir = manager + .read_section_directory() + .unwrap() + .expect("directory"); + assert_eq!( + section_dir.find(SectionType::PropertyIndex).unwrap().version, + 1 + ); + manager.close().unwrap(); +} diff --git a/crates/grafeo-storage/src/container/directory.rs b/crates/grafeo-storage/src/container/directory.rs index f32997b46..9d645343a 100644 --- a/crates/grafeo-storage/src/container/directory.rs +++ b/crates/grafeo-storage/src/container/directory.rs @@ -148,8 +148,12 @@ impl SectionDirectory { let mut entries = Vec::with_capacity(count); for i in 0..count { let offset = 8 + i * SectionDirectoryEntry::SIZE; - let entry = read_entry(&data[offset..offset + SectionDirectoryEntry::SIZE])?; - entries.push(entry); + // Unknown optional section types are skipped so older binaries can + // open files that carry newer optional indexes. Unknown required + // types fail closed (see `read_entry`). + if let Some(entry) = read_entry(&data[offset..offset + SectionDirectoryEntry::SIZE])? { + entries.push(entry); + } } Ok(Self { entries }) @@ -182,8 +186,14 @@ fn write_entry(buf: &mut [u8], entry: &SectionDirectoryEntry) { buf[28..32].copy_from_slice(&[0, 0, 0, 0]); // reserved } -fn read_entry(buf: &[u8]) -> Result { +/// Parse one directory entry. +/// +/// Returns `Ok(None)` when the type id is unknown and the entry is optional +/// (`flags.required == false`), so older readers can skip newer optional +/// sections. Returns an error when the type is unknown and required. +fn read_entry(buf: &[u8]) -> Result> { let type_val = u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]); + let flags = SectionFlags::from_byte(buf[5]); let section_type = match type_val { 1 => SectionType::Catalog, 2 => SectionType::LpgStore, @@ -195,20 +205,24 @@ fn read_entry(buf: &[u8]) -> Result { 12 => SectionType::RdfRing, 20 => SectionType::PropertyIndex, other => { - return Err(Error::Serialization(format!( - "unknown section type: {other}" - ))); + if flags.required { + return Err(Error::Serialization(format!( + "unknown required section type: {other}" + ))); + } + // Optional unknown: skip without failing open. + return Ok(None); } }; - Ok(SectionDirectoryEntry { + Ok(Some(SectionDirectoryEntry { section_type, version: buf[4], - flags: SectionFlags::from_byte(buf[5]), + flags, offset: u64::from_le_bytes(buf[8..16].try_into().unwrap()), length: u64::from_le_bytes(buf[16..24].try_into().unwrap()), checksum: u32::from_le_bytes(buf[24..28].try_into().unwrap()), - }) + })) } #[cfg(test)] @@ -435,17 +449,55 @@ mod tests { } #[test] - fn from_bytes_unknown_section_type() { + fn from_bytes_unknown_optional_section_type_is_skipped() { let mut buf = vec![0u8; DIRECTORY_PAGE_SIZE]; - // 1 entry + // 2 entries: known Catalog + unknown optional type 99 + buf[0..4].copy_from_slice(&2u32.to_le_bytes()); + + // Entry 0: Catalog (required), version 2 + let e0 = 8; + buf[e0..e0 + 4].copy_from_slice(&(SectionType::Catalog as u32).to_le_bytes()); + buf[e0 + 4] = 2; + buf[e0 + 5] = SectionType::Catalog.default_flags().to_byte(); + buf[e0 + 8..e0 + 16].copy_from_slice(&SECTION_DATA_OFFSET.to_le_bytes()); + buf[e0 + 16..e0 + 24].copy_from_slice(&64u64.to_le_bytes()); + buf[e0 + 24..e0 + 28].copy_from_slice(&0u32.to_le_bytes()); + + // Entry 1: unknown type 99, optional (required bit clear) + let e1 = 8 + SectionDirectoryEntry::SIZE; + buf[e1..e1 + 4].copy_from_slice(&99u32.to_le_bytes()); + buf[e1 + 4] = 1; + buf[e1 + 5] = SectionFlags { + required: false, + mmap_able: true, + } + .to_byte(); + buf[e1 + 8..e1 + 16].copy_from_slice(&(SECTION_DATA_OFFSET + 4096).to_le_bytes()); + buf[e1 + 16..e1 + 24].copy_from_slice(&128u64.to_le_bytes()); + buf[e1 + 24..e1 + 28].copy_from_slice(&1u32.to_le_bytes()); + + let dir = SectionDirectory::from_bytes(&buf).expect("optional unknown must not fail open"); + assert_eq!(dir.len(), 1, "optional unknown entry must be skipped"); + assert!(dir.find(SectionType::Catalog).is_some()); + } + + #[test] + fn from_bytes_unknown_required_section_type_fails_closed() { + let mut buf = vec![0u8; DIRECTORY_PAGE_SIZE]; + // 1 entry: unknown type 99 with required flag set buf[0..4].copy_from_slice(&1u32.to_le_bytes()); - // Write an unknown section type (99) at entry offset buf[8..12].copy_from_slice(&99u32.to_le_bytes()); + buf[12] = 1; // version + buf[13] = SectionFlags { + required: true, + mmap_able: false, + } + .to_byte(); let result = SectionDirectory::from_bytes(&buf); assert!(result.is_err()); let err = result.unwrap_err().to_string(); assert!( - err.contains("unknown section type"), + err.contains("unknown required section type") && err.contains("99"), "unexpected error: {err}" ); } diff --git a/crates/grafeo-storage/src/file/manager.rs b/crates/grafeo-storage/src/file/manager.rs index beb4f7a84..3d0480701 100644 --- a/crates/grafeo-storage/src/file/manager.rs +++ b/crates/grafeo-storage/src/file/manager.rs @@ -420,8 +420,36 @@ impl GrafeoFileManager { // ── Section-based I/O (v2 container format) ───────────────────── + /// Writes multiple sections with directory version fixed to `1` for each. + /// + /// Prefer [`Self::write_versioned_sections`] when the caller knows each + /// section's declared format version (engine flush path). This legacy + /// entry point remains for tests and callers that only have opaque bytes. + /// + /// # Errors + /// + /// Returns an error if write or sync fails. + pub fn write_sections( + &self, + sections: &[(grafeo_common::storage::SectionType, &[u8])], + epoch: u64, + transaction_id: u64, + node_count: u64, + edge_count: u64, + ) -> Result<()> { + let versioned: Vec<(grafeo_common::storage::SectionType, u8, &[u8])> = sections + .iter() + .map(|(section_type, data)| (*section_type, 1u8, *data)) + .collect(); + self.write_versioned_sections(&versioned, epoch, transaction_id, node_count, edge_count) + } + /// Writes multiple sections to the file using the v2 container format. /// + /// Each tuple is `(section_type, directory_version, payload)`. The directory + /// entry records the supplied `directory_version` (the section's declared + /// format version) rather than a hard-coded `1`. + /// /// Each section is written at a page-aligned offset. A section directory /// is written at `DIRECTORY_OFFSET`, and a new DbHeader is committed to /// the inactive slot. @@ -429,9 +457,9 @@ impl GrafeoFileManager { /// # Errors /// /// Returns an error if write or sync fails. - pub fn write_sections( + pub fn write_versioned_sections( &self, - sections: &[(grafeo_common::storage::SectionType, &[u8])], + sections: &[(grafeo_common::storage::SectionType, u8, &[u8])], epoch: u64, transaction_id: u64, node_count: u64, @@ -466,7 +494,7 @@ impl GrafeoFileManager { #[allow(clippy::cast_possible_truncation)] let nonce_iteration = (active_header.iteration + 1) as u32; - for (section_type, data) in sections { + for (section_type, version, data) in sections { // Encrypt section data if encryption is enabled. // Nonce high word: iteration in bits [31:8], section type in bits [7:0]. // Bit-packing (not XOR) ensures unique high words: XOR is commutative @@ -502,7 +530,7 @@ impl GrafeoFileManager { dir.upsert(SectionDirectoryEntry { section_type: *section_type, - version: 1, + version: *version, flags: section_type.default_flags(), offset: current_offset, length, @@ -1654,4 +1682,175 @@ mod tests { assert!(result.is_err(), "decryption with wrong key should fail"); } } + + // ── G-F0.1: truthful per-section directory versions ───────────── + + #[test] + fn write_versioned_sections_preserves_supplied_versions() { + use grafeo_common::storage::SectionType; + + let dir = test_dir(); + let path = dir.path().join("versioned.grafeo"); + + let manager = GrafeoFileManager::create(&path).unwrap(); + manager + .write_versioned_sections( + &[ + (SectionType::Catalog, 2, b"catalog-v2".as_slice()), + (SectionType::LpgStore, 2, b"lpg-v2".as_slice()), + (SectionType::CompactStore, 3, b"compact-v3".as_slice()), + (SectionType::VectorStore, 2, b"vector-v2".as_slice()), + (SectionType::TextIndex, 1, b"text-v1".as_slice()), + (SectionType::PropertyIndex, 1, b"prop-v1".as_slice()), + ], + 1, + 1, + 0, + 0, + ) + .unwrap(); + + let section_dir = manager + .read_section_directory() + .unwrap() + .expect("directory should exist"); + + let expected = [ + (SectionType::Catalog, 2u8, b"catalog-v2".as_slice()), + (SectionType::LpgStore, 2, b"lpg-v2"), + (SectionType::CompactStore, 3, b"compact-v3"), + (SectionType::VectorStore, 2, b"vector-v2"), + (SectionType::TextIndex, 1, b"text-v1"), + (SectionType::PropertyIndex, 1, b"prop-v1"), + ]; + for (section_type, version, payload) in expected { + let entry = section_dir + .find(section_type) + .unwrap_or_else(|| panic!("missing {section_type:?}")); + assert_eq!( + entry.version, version, + "{section_type:?} directory version" + ); + let data = manager.read_section_data(entry).unwrap(); + assert_eq!(data, payload, "{section_type:?} payload"); + } + manager.close().unwrap(); + } + + #[test] + fn write_sections_still_records_directory_version_one() { + use grafeo_common::storage::SectionType; + + let dir = test_dir(); + let path = dir.path().join("legacy_write.grafeo"); + + let manager = GrafeoFileManager::create(&path).unwrap(); + manager + .write_sections( + &[ + (SectionType::LpgStore, b"legacy".as_slice()), + (SectionType::VectorStore, b"vec".as_slice()), + ], + 1, + 1, + 0, + 0, + ) + .unwrap(); + + let section_dir = manager + .read_section_directory() + .unwrap() + .expect("directory should exist"); + assert_eq!(section_dir.find(SectionType::LpgStore).unwrap().version, 1); + assert_eq!( + section_dir.find(SectionType::VectorStore).unwrap().version, + 1 + ); + manager.close().unwrap(); + } + + #[test] + fn historical_outer_v1_with_higher_payload_bytes_remains_readable() { + // Historical writers recorded directory version 1 even when the + // payload body was a later format. Readers must not reject that + // outer-vs-payload mismatch (no strict equality gate). + use grafeo_common::storage::SectionType; + + let dir = test_dir(); + let path = dir.path().join("outer_v1_payload.grafeo"); + + // Simulate historical outer-v1: use write_sections (always v1) with + // bytes that a modern CompactStore/LPG payload reader would treat as + // its own higher internal version. Storage only needs the bytes + // round-trip; payload dispatch is covered by engine/core tests. + let payload = b"pretend-v2-or-v3-payload-body"; + { + let manager = GrafeoFileManager::create(&path).unwrap(); + manager + .write_sections(&[(SectionType::CompactStore, payload.as_slice())], 1, 1, 0, 0) + .unwrap(); + manager.close().unwrap(); + } + + let manager = GrafeoFileManager::open(&path).unwrap(); + let section_dir = manager + .read_section_directory() + .unwrap() + .expect("directory should exist"); + let entry = section_dir.find(SectionType::CompactStore).unwrap(); + assert_eq!( + entry.version, 1, + "historical outer directory version stays 1" + ); + assert_eq!(manager.read_section_data(entry).unwrap(), payload); + manager.close().unwrap(); + } + + #[test] + #[cfg(all(feature = "encryption", not(miri)))] + fn encrypted_write_versioned_sections_preserves_versions() { + use grafeo_common::encryption::KeyChain; + use grafeo_common::storage::SectionType; + + let dir = test_dir(); + let path = dir.path().join("encrypted_versions.grafeo"); + let kc = KeyChain::new([0xCD; 32]); + + { + let mut manager = GrafeoFileManager::create(&path).unwrap(); + manager.set_section_encryptor(kc.encryptor_for("section", b"test")); + manager + .write_versioned_sections( + &[ + (SectionType::Catalog, 2, b"enc-catalog".as_slice()), + (SectionType::LpgStore, 2, b"enc-lpg".as_slice()), + (SectionType::VectorStore, 2, b"enc-vec".as_slice()), + ], + 1, + 0, + 0, + 0, + ) + .unwrap(); + manager.close().unwrap(); + } + + let mut manager = GrafeoFileManager::open(&path).unwrap(); + manager.set_section_encryptor(kc.encryptor_for("section", b"test")); + let section_dir = manager + .read_section_directory() + .unwrap() + .expect("directory should exist"); + for (section_type, version, payload) in [ + (SectionType::Catalog, 2u8, b"enc-catalog".as_slice()), + (SectionType::LpgStore, 2, b"enc-lpg"), + (SectionType::VectorStore, 2, b"enc-vec"), + ] { + let entry = section_dir.find(section_type).unwrap(); + assert_eq!(entry.version, version, "{section_type:?}"); + assert_eq!(manager.read_section_data(entry).unwrap(), payload); + } + manager.close().unwrap(); + } } diff --git a/docs/architecture/storage/container-format.md b/docs/architecture/storage/container-format.md index dc51bfe7c..614ab6570 100644 --- a/docs/architecture/storage/container-format.md +++ b/docs/architecture/storage/container-format.md @@ -88,7 +88,7 @@ Maximum capacity: 127 sections (`(4096 - 8) / 32`). | Offset | Size | Type | Field | Description | |--------|------|------|-------|-------------| | 0 | 4 | `u32 LE` | `section_type` | Section type ID (see table below) | -| 4 | 1 | `u8` | `version` | Per-section format version | +| 4 | 1 | `u8` | `version` | Per-section format version (see Truthful directory versions) | | 5 | 1 | `u8` | `flags` | Bit 0: required, Bit 1: mmap-able | | 6 | 2 | `u16 LE` | `reserved` | Zero | | 8 | 8 | `u64 LE` | `offset` | Byte offset from file start | @@ -98,19 +98,45 @@ Maximum capacity: 127 sections (`(4096 - 8) / 32`). Remaining bytes after the last entry are zero-filled to 4 KiB. +### Truthful directory versions (G-F0.1) + +The directory entry `version` byte is the per-section format version declared +by that section's `Section::version()` implementation. + +- **New writers** (`GrafeoFileManager::write_versioned_sections`, used by + engine flush) record each section's declared version in the directory. +- **Legacy callers** of `write_sections` still emit directory version `1` for + every section (tests and opaque-byte writers). +- **Historical caveat:** shared writers prior to G-F0.1 often recorded outer + directory version `1` for every section even when the payload itself was a + later format (for example CompactStore payload v2/v3 with outer entry v1). + Readers must continue to dispatch from the **payload** header (for example + the CompactStore `GCST` version byte), not require outer directory version + equals payload version. Do **not** add a strict outer-equals-payload gate + that would reject valid historical files. +- **Unknown section types:** if the type id is not recognized and + `flags.required` is clear, the entry is skipped so older binaries can open + files that carry newer optional indexes. If `flags.required` is set, open + fails closed. + --- ## Section Types +Values and default flags match `SectionType` / `SectionType::default_flags` +in `grafeo-common`. + | Value | Name | Required | Mmap-able | Description | |-------|------|----------|-----------|-------------| -| 1 | `CATALOG` | yes | no | Schema defs, index metadata, epoch, config | -| 2 | `LPG_STORE` | yes | no | Nodes, edges, properties, named graphs | -| 3 | `RDF_STORE` | no | no | RDF triples, named graphs | -| 10 | `VECTOR_STORE` | no | yes | Embeddings + HNSW topology | -| 11 | `TEXT_INDEX` | no | yes | BM25 postings + term dictionary | -| 12 | `RDF_RING` | no | yes | Wavelet trees + dictionary | -| 20 | `PROPERTY_INDEX` | no | yes | Property hash/btree indexes | +| 1 | `Catalog` | yes | no | Schema defs, index metadata, epoch, config | +| 2 | `LpgStore` | yes | no | Nodes, edges, properties, named graphs | +| 3 | `RdfStore` | no | no | RDF triples, named graphs | +| 4 | `CompactStore` | yes | yes | Columnar compact base (`GCST` payload) | +| 5 | `OverlayDeletions` | no | no | Layered base-deletion tombstones | +| 10 | `VectorStore` | no | yes | Embeddings + HNSW topology | +| 11 | `TextIndex` | no | yes | BM25 postings + term dictionary | +| 12 | `RdfRing` | no | yes | Wavelet trees + dictionary | +| 20 | `PropertyIndex` | no | yes | Property hash/btree indexes | **Type ranges:** @@ -123,12 +149,13 @@ Remaining bytes after the last entry are zero-filled to 4 KiB. - **Bit 0 (required):** If set, older binaries that don't recognize this section type must refuse to open the file. If clear, the section can be safely skipped (the database opens without that index). -- **Bit 1 (mmap-able):** If set, the section uses a fixed binary layout - suitable for zero-copy memory-mapped access. If clear, the section must - be deserialized into RAM (bincode format). +- **Bit 1 (mmap-able):** Capability flag: the section layout is intended to + support memory-mapped access. It does **not** mean the current container + open path always mmaps that section. **Empty sections** are omitted from the directory entirely. If no RDF data -exists, there is no `RDF_STORE` entry. +exists, there is no `RdfStore` entry. CompactStore appears only after +`compact()` (or an equivalent layered write). --- @@ -145,19 +172,21 @@ boundary after the previous section ends. ... ``` -### Data Section Encoding (Catalog, LPG, RDF) +### Per-type encoding -Data sections use **bincode** serialization (standard configuration). They -are fully deserialized into RAM on load. The internal format is -version-specific (the `version` byte in the directory entry allows -independent evolution). +Encoding is **not** uniformly bincode: -### Index Section Encoding (Vector, Text, Ring, Property) +| Section | Encoding | Notes | +|---------|----------|-------| +| `Catalog`, `LpgStore`, `RdfStore` | bincode | Fully deserialized into RAM on ordinary open | +| `CompactStore` | Custom **`GCST`** payload | Column codecs + section-level strings; CRC32 trailer | +| `OverlayDeletions` | dedicated deletions codec | Base tombstones for layered reopen | +| `VectorStore`, `TextIndex`, `RdfRing`, `PropertyIndex` | section-specific codecs | `mmap_able` marks intended zero-copy layouts | -Index sections use **bincode** serialization currently (version 1). Future -versions may switch to fixed binary layouts for zero-copy mmap access. -The `mmap_able` flag indicates whether the section can be memory-mapped -after being written. +Payload version dispatch is owned by each section deserializer. Directory +version metadata is advisory for tooling and independent evolution; historical +outer-v1 files with supported payloads remain readable (see Truthful directory +versions above). --- From 1f751928e07a65ce632892e7a1a0ae745c49c7d7 Mon Sep 17 00:00:00 2001 From: jarmen423 Date: Thu, 30 Jul 2026 21:22:31 +0000 Subject: [PATCH 2/2] style: rustfmt G-F0.1 section directory version files Apply rustfmt --edition 2024 to the four files that failed targeted format check. Formatting-only; no semantic changes. --- crates/grafeo-engine/src/database/flush.rs | 5 +---- .../tests/section_directory_versions.rs | 22 ++++++++++--------- crates/grafeo-storage/src/file/manager.rs | 13 ++++++----- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/crates/grafeo-engine/src/database/flush.rs b/crates/grafeo-engine/src/database/flush.rs index 556ecf4bb..c482a8d6f 100644 --- a/crates/grafeo-engine/src/database/flush.rs +++ b/crates/grafeo-engine/src/database/flush.rs @@ -103,10 +103,7 @@ pub(super) fn flush( // Mark all written sections as clean for section in sections { - if targets - .iter() - .any(|(t, _, _)| *t == section.section_type()) - { + if targets.iter().any(|(t, _, _)| *t == section.section_type()) { section.mark_clean(); } } diff --git a/crates/grafeo-engine/tests/section_directory_versions.rs b/crates/grafeo-engine/tests/section_directory_versions.rs index fb4a6c386..cd5c363e3 100644 --- a/crates/grafeo-engine/tests/section_directory_versions.rs +++ b/crates/grafeo-engine/tests/section_directory_versions.rs @@ -121,10 +121,8 @@ fn historical_outer_v1_compact_payload_still_opens() { let data = manager.read_section_data(entry).unwrap(); rewritten.push((entry.section_type, data)); } - let refs: Vec<(SectionType, &[u8])> = rewritten - .iter() - .map(|(t, d)| (*t, d.as_slice())) - .collect(); + let refs: Vec<(SectionType, &[u8])> = + rewritten.iter().map(|(t, d)| (*t, d.as_slice())).collect(); // write_sections always records directory version 1 (legacy path). manager .write_sections(&refs, 1, 1, 1, 0) @@ -202,10 +200,7 @@ fn checkpoint_records_vector_store_declared_version() { } let declared = VectorStoreSection::new(Vec::new()).version(); - assert_eq!( - directory_version(&path, SectionType::VectorStore), - declared - ); + assert_eq!(directory_version(&path, SectionType::VectorStore), declared); } #[test] @@ -242,7 +237,11 @@ fn property_index_directory_version_via_shared_writer() { let manager = GrafeoFileManager::create(&path).unwrap(); manager .write_versioned_sections( - &[(SectionType::PropertyIndex, 1, b"prop-index-payload".as_slice())], + &[( + SectionType::PropertyIndex, + 1, + b"prop-index-payload".as_slice(), + )], 1, 1, 0, @@ -254,7 +253,10 @@ fn property_index_directory_version_via_shared_writer() { .unwrap() .expect("directory"); assert_eq!( - section_dir.find(SectionType::PropertyIndex).unwrap().version, + section_dir + .find(SectionType::PropertyIndex) + .unwrap() + .version, 1 ); manager.close().unwrap(); diff --git a/crates/grafeo-storage/src/file/manager.rs b/crates/grafeo-storage/src/file/manager.rs index 3d0480701..d978d8954 100644 --- a/crates/grafeo-storage/src/file/manager.rs +++ b/crates/grafeo-storage/src/file/manager.rs @@ -1727,10 +1727,7 @@ mod tests { let entry = section_dir .find(section_type) .unwrap_or_else(|| panic!("missing {section_type:?}")); - assert_eq!( - entry.version, version, - "{section_type:?} directory version" - ); + assert_eq!(entry.version, version, "{section_type:?} directory version"); let data = manager.read_section_data(entry).unwrap(); assert_eq!(data, payload, "{section_type:?} payload"); } @@ -1788,7 +1785,13 @@ mod tests { { let manager = GrafeoFileManager::create(&path).unwrap(); manager - .write_sections(&[(SectionType::CompactStore, payload.as_slice())], 1, 1, 0, 0) + .write_sections( + &[(SectionType::CompactStore, payload.as_slice())], + 1, + 1, + 0, + 0, + ) .unwrap(); manager.close().unwrap(); }