From e1a54c2515a0145dd685008f6a7e0eab5f773fdc Mon Sep 17 00:00:00 2001 From: SWORDIntel Date: Mon, 13 Jul 2026 23:14:52 +0100 Subject: [PATCH 1/5] feat: persist typed vector graph edges --- Makefile | 8 +- include/qihse_vector_db.h | 76 +++ persistence/qihse_container.h | 3 +- persistence/qihse_vector_store.c | 9 +- persistence/qihse_vector_store.h | 2 + python/qihse/core.py | 134 ++++- src/broad_oak/qihse_vector_db.c | 857 ++++++++++++++++++++++++++-- tests/qihse_edge_persistence_test.c | 143 +++++ tests/test_python_edges.py | 48 ++ 9 files changed, 1223 insertions(+), 57 deletions(-) create mode 100644 tests/qihse_edge_persistence_test.c create mode 100644 tests/test_python_edges.py diff --git a/Makefile b/Makefile index 2b65bc2..ef4c1ed 100644 --- a/Makefile +++ b/Makefile @@ -146,7 +146,7 @@ endif # because their functionality is already partially in qihse_math.c / qihse_search.c # or provided by qihse_exports.c stubs. -.PHONY: all build build-native clean pristine workspace workspace-clean lib lib-ctypes liboqs oqs-provider persistence persistence-check test benchmark install dev-setup docs test-persist test-trinary-codec test-memory-planner test-memory-topology-probe test-memory-planner-trace test-memory-allocation-policy test-memory-coherence test-memory-migration-policy test-memory-migration test-memory-device-placement test-memory-migration-backend test-memory-migration-scheduler bench-trinary-codec bench-trinary-db-candidate bench-micro bench-trinary-search-path bench-trinary-search-sweep bench-trinary-random-sweep bench-trinary-weighted-sweep bench-trinary-magnitude-sweep bench-reference-workloads bench-reference-runner-smoke sample-vxug-pdf-workload bench-vxug-pdf-workload bench-reference-workload bench-reference-result-summary bench-sift1m-workload bench-sift1m-fallback-data calibrate-sift1m-workload validate-reference-workflow check-upstream-workflow check-upstream-workflow-strict check upstream-pr-loop test-all-isa test-vnni-bench test-vnni-only test-avx2-only test-avx512-direct test-amx-only test-direct-execution test-simple-exec +.PHONY: all build build-native clean pristine workspace workspace-clean lib lib-ctypes liboqs oqs-provider persistence persistence-check test benchmark install dev-setup docs test-persist test-edge-persistence test-trinary-codec test-memory-planner test-memory-topology-probe test-memory-planner-trace test-memory-allocation-policy test-memory-coherence test-memory-migration-policy test-memory-migration test-memory-device-placement test-memory-migration-backend test-memory-migration-scheduler bench-trinary-codec bench-trinary-db-candidate bench-micro bench-trinary-search-path bench-trinary-search-sweep bench-trinary-random-sweep bench-trinary-weighted-sweep bench-trinary-magnitude-sweep bench-reference-workloads bench-reference-runner-smoke sample-vxug-pdf-workload bench-vxug-pdf-workload bench-reference-workload bench-reference-result-summary bench-sift1m-workload bench-sift1m-fallback-data calibrate-sift1m-workload validate-reference-workflow check-upstream-workflow check-upstream-workflow-strict check upstream-pr-loop test-all-isa test-vnni-bench test-vnni-only test-avx2-only test-avx512-direct test-amx-only test-direct-execution test-simple-exec .NOTPARALLEL: validate-reference-workflow all: liboqs oqs-provider lib server keygen @@ -225,6 +225,12 @@ test-persist: lib -L. -lqihse $(LDFLAGS) LD_LIBRARY_PATH=. ./tests/qihse_vector_db_persistence_test +test-edge-persistence: lib + $(CC) $(CFLAGS) -o tests/qihse_edge_persistence_test \ + tests/qihse_edge_persistence_test.c \ + -L. -lqihse $(LDFLAGS) + LD_LIBRARY_PATH=. ./tests/qihse_edge_persistence_test + test: test-omni test-e2e test-e2e-memory-planner test-persist test-bytecode test-document-store test-column-store test-fts-engine test-timeseries test-trinary-codec test-memory-planner test-memory-topology-probe test-memory-planner-trace test-memory-allocation-policy test-memory-coherence test-memory-migration-policy test-memory-migration test-memory-device-placement test-memory-migration-backend test-memory-migration-scheduler test-quantization test-bytecode: lib diff --git a/include/qihse_vector_db.h b/include/qihse_vector_db.h index 5d758d4..e6c74f1 100644 --- a/include/qihse_vector_db.h +++ b/include/qihse_vector_db.h @@ -532,6 +532,9 @@ bool qihse_vector_db_add_model_weights( * * @param vdb Vector database handle * @param vector_id External vector ID to delete + * Deletion fails with EBUSY while explicit edges reference the vector; remove + * those relationships first so every persisted edge retains valid endpoints. + * * @return true if a live vector was deleted, false on failure or missing ID */ bool qihse_vector_db_delete_by_id( @@ -721,6 +724,30 @@ bool qihse_vector_db_search_batch( * EXPLICIT GRAPH EDGE MANAGEMENT (QQL/Graph DB) * ============================================================================ */ +#define QIHSE_EDGE_TYPE_MAX 31u + +typedef enum qihse_edge_direction_e { + QIHSE_EDGE_OUTGOING = 0, + QIHSE_EDGE_INCOMING = 1, + QIHSE_EDGE_BOTH = 2 +} qihse_edge_direction_t; + +typedef struct qihse_edge_input_s { + uint64_t from_id; + uint64_t to_id; + const char* edge_type; + const void* metadata; + size_t metadata_size; +} qihse_edge_input_t; + +typedef struct qihse_edge_result_s { + uint64_t from_id; + uint64_t to_id; + char edge_type[QIHSE_EDGE_TYPE_MAX + 1u]; + void* metadata; + size_t metadata_size; +} qihse_edge_result_t; + /** * Add an explicit edge between two vector nodes. * @@ -759,6 +786,55 @@ int qihse_vector_db_get_edges( size_t max_edges ); +/** Add or idempotently retain a batch of typed edges. */ +bool qihse_vector_db_add_edges( + qihse_vector_db_t vdb, + const qihse_edge_input_t* edges, + size_t edge_count, + size_t* changed_count +); + +/** Replace metadata on an existing typed edge. */ +bool qihse_vector_db_replace_edge( + qihse_vector_db_t vdb, + uint64_t from_id, + uint64_t to_id, + const char* edge_type, + const void* metadata, + size_t metadata_size +); + +/** Remove an existing typed edge. Missing edges are idempotent success. */ +bool qihse_vector_db_remove_edge( + qihse_vector_db_t vdb, + uint64_t from_id, + uint64_t to_id, + const char* edge_type +); + +/** Retrieve typed neighbors in the requested direction. */ +int qihse_vector_db_get_typed_neighbors( + qihse_vector_db_t vdb, + uint64_t node_id, + const char* edge_type, + qihse_edge_direction_t direction, + uint64_t* out_ids, + size_t max_edges +); + +/** Retrieve edge records including owned metadata copies. */ +int qihse_vector_db_get_edge_records( + qihse_vector_db_t vdb, + uint64_t node_id, + const char* edge_type, + qihse_edge_direction_t direction, + qihse_edge_result_t* results, + size_t max_edges +); + +/** Release metadata allocated by qihse_vector_db_get_edge_records. */ +void qihse_vector_db_free_edge_records(qihse_edge_result_t* results, size_t count); + /* ============================================================================ * EMBEDDED QUERY EXECUTION (QQL & SQL) * ============================================================================ */ diff --git a/persistence/qihse_container.h b/persistence/qihse_container.h index ecf3cef..4244e4b 100644 --- a/persistence/qihse_container.h +++ b/persistence/qihse_container.h @@ -35,10 +35,11 @@ extern "C" { #define QIHSE_CTR_SEC_GRAPH 0x0009u #define QIHSE_CTR_SEC_INT8 0x000Au #define QIHSE_CTR_SEC_TIER 0x000Bu +#define QIHSE_CTR_SEC_EDGES 0x000Cu #define QIHSE_CTR_SEC_KEY 0x1000u #define QIHSE_CTR_SEC_SIGNATURE 0x1001u -#define QIHSE_CTR_NUM_SECTIONS 13u +#define QIHSE_CTR_NUM_SECTIONS 14u /* ── Layout constants ─────────────────────────────────────────────── */ #define QIHSE_CTR_MAGIC "QIHSEQDB" diff --git a/persistence/qihse_vector_store.c b/persistence/qihse_vector_store.c index 97ed126..1dcda96 100644 --- a/persistence/qihse_vector_store.c +++ b/persistence/qihse_vector_store.c @@ -819,7 +819,8 @@ bool qihse_vector_store_flush(const char* db_path, const qihse_vector_store_flus (!in->metadata && in->metadata_bytes != 0u) || (!in->idmap && in->idmap_count != 0u) || (!in->trinary && in->trinary_bytes != 0u) || - (!in->magnitude && in->magnitude_bytes != 0u)) { + (!in->magnitude && in->magnitude_bytes != 0u) || + (!in->explicit_edges && in->explicit_edges_bytes != 0u)) { errno = EINVAL; return false; } @@ -909,7 +910,7 @@ bool qihse_vector_store_flush(const char* db_path, const qihse_vector_store_flus { /* Build the section buffer list for the atomic container flush. * Order: MANIFEST last so a partial write can be detected on reopen. */ - qihse_ctr_section_buf_t bufs[8]; + qihse_ctr_section_buf_t bufs[9]; size_t nb = 0u; bufs[nb].section_id = QIHSE_CTR_SEC_VECTORS; bufs[nb].data = in->vectors; @@ -939,6 +940,10 @@ bool qihse_vector_store_flush(const char* db_path, const qihse_vector_store_flus bufs[nb].size = in->magnitude_bytes; nb++; } + bufs[nb].section_id = QIHSE_CTR_SEC_EDGES; + bufs[nb].data = in->explicit_edges; + bufs[nb].size = in->explicit_edges_bytes; + nb++; bufs[nb].section_id = QIHSE_CTR_SEC_MANIFEST; bufs[nb].data = manifest_data; bufs[nb].size = sizeof(manifest_data); diff --git a/persistence/qihse_vector_store.h b/persistence/qihse_vector_store.h index 647b417..be3c2fc 100644 --- a/persistence/qihse_vector_store.h +++ b/persistence/qihse_vector_store.h @@ -120,6 +120,8 @@ typedef struct qihse_vector_store_flush_s { uint64_t magnitude_generation; uint64_t magnitude_row_bytes; uint32_t magnitude_flags; + const void* explicit_edges; + size_t explicit_edges_bytes; } qihse_vector_store_flush_t; bool qihse_vector_store_load(const char* db_path, qihse_vector_store_snapshot_t* out); diff --git a/python/qihse/core.py b/python/qihse/core.py index f2f0156..877ec05 100644 --- a/python/qihse/core.py +++ b/python/qihse/core.py @@ -57,6 +57,8 @@ class DistanceMetric(IntEnum): _lib.qihse_vector_db_destroy.argtypes = [_VectorDB_p] _lib.qihse_vector_db_destroy.restype = None +_lib.qihse_vector_db_close.argtypes = [_VectorDB_p] +_lib.qihse_vector_db_close.restype = ctypes.c_bool _lib.qihse_vector_db_add_vectors.argtypes = [ _VectorDB_p, @@ -96,6 +98,33 @@ class CVectorResult(ctypes.Structure): ("metadata_size", ctypes.c_size_t), ] +class CEdgeInput(ctypes.Structure): + _fields_ = [ + ("from_id", ctypes.c_uint64), ("to_id", ctypes.c_uint64), + ("edge_type", ctypes.c_char_p), ("metadata", ctypes.c_void_p), + ("metadata_size", ctypes.c_size_t), + ] + +class CEdgeResult(ctypes.Structure): + _fields_ = [ + ("from_id", ctypes.c_uint64), ("to_id", ctypes.c_uint64), + ("edge_type", ctypes.c_char * 32), ("metadata", ctypes.c_void_p), + ("metadata_size", ctypes.c_size_t), + ] + +_lib.qihse_vector_db_add_edges.argtypes = [_VectorDB_p, ctypes.POINTER(CEdgeInput), ctypes.c_size_t, ctypes.POINTER(ctypes.c_size_t)] +_lib.qihse_vector_db_add_edges.restype = ctypes.c_bool +_lib.qihse_vector_db_replace_edge.argtypes = [_VectorDB_p, ctypes.c_uint64, ctypes.c_uint64, ctypes.c_char_p, ctypes.c_void_p, ctypes.c_size_t] +_lib.qihse_vector_db_replace_edge.restype = ctypes.c_bool +_lib.qihse_vector_db_remove_edge.argtypes = [_VectorDB_p, ctypes.c_uint64, ctypes.c_uint64, ctypes.c_char_p] +_lib.qihse_vector_db_remove_edge.restype = ctypes.c_bool +_lib.qihse_vector_db_get_typed_neighbors.argtypes = [_VectorDB_p, ctypes.c_uint64, ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_uint64), ctypes.c_size_t] +_lib.qihse_vector_db_get_typed_neighbors.restype = ctypes.c_int +_lib.qihse_vector_db_get_edge_records.argtypes = [_VectorDB_p, ctypes.c_uint64, ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(CEdgeResult), ctypes.c_size_t] +_lib.qihse_vector_db_get_edge_records.restype = ctypes.c_int +_lib.qihse_vector_db_free_edge_records.argtypes = [ctypes.POINTER(CEdgeResult), ctypes.c_size_t] +_lib.qihse_vector_db_free_edge_records.restype = None + _lib.qihse_vector_db_search.argtypes = [ _VectorDB_p, ctypes.POINTER(CVectorQuery), @@ -161,9 +190,9 @@ def create(path: str, dims: int) -> "VectorDB": @staticmethod def open(path: str, read_only: bool = False) -> "VectorDB": - flags = 0 + flags = 0x00000008 if read_only: - flags |= 0x00000001 + flags |= 0x00000002 ptr = _lib.qihse_vector_db_open(0, None, path.encode("utf-8"), flags) if not ptr: raise RuntimeError(f"Failed to open VectorDB at {path}") @@ -172,7 +201,9 @@ def open(path: str, read_only: bool = False) -> "VectorDB": def close(self): with self._lock: if self._ptr: - _lib.qihse_vector_db_destroy(self._ptr) + if not _lib.qihse_vector_db_close(self._ptr): + self._ptr = None + raise RuntimeError("Failed to flush and close VectorDB") self._ptr = None def __enter__(self): @@ -270,6 +301,103 @@ def search( results.append(VectorResult(int(out_results[i].id), float(out_results[i].score))) return results + @staticmethod + def _edge_type_bytes(edge_type: str) -> bytes: + if not isinstance(edge_type, str): + raise TypeError("edge_type must be a string") + encoded = edge_type.encode("utf-8") + if not encoded or len(encoded) > 31 or b"\x00" in encoded: + raise ValueError("edge_type must encode to 1..31 non-NUL bytes") + return encoded + + def add_edges(self, edges: list[tuple[int, int, str, Optional[bytes]]]) -> int: + if not isinstance(edges, list) or not edges: + raise ValueError("edges must be a non-empty list") + with self._lock: + c_edges = (CEdgeInput * len(edges))() + keepalive = [] + for index, edge in enumerate(edges): + if not isinstance(edge, tuple) or len(edge) != 4: + raise ValueError("each edge must be (from_id, to_id, edge_type, metadata)") + from_id, to_id, edge_type, metadata = edge + encoded_type = self._edge_type_bytes(edge_type) + if metadata is not None and not isinstance(metadata, bytes): + raise TypeError("edge metadata must be bytes or None") + c_edges[index].from_id = from_id + c_edges[index].to_id = to_id + c_edges[index].edge_type = encoded_type + keepalive.append(encoded_type) + if metadata: + buffer = ctypes.create_string_buffer(metadata) + keepalive.append(buffer) + c_edges[index].metadata = ctypes.cast(buffer, ctypes.c_void_p) + c_edges[index].metadata_size = len(metadata) + changed = ctypes.c_size_t() + if not _lib.qihse_vector_db_add_edges(self._ptr, c_edges, len(edges), ctypes.byref(changed)): + raise RuntimeError("failed to add edges") + return changed.value + + def add_edge(self, from_id: int, to_id: int, edge_type: str, + metadata: Optional[bytes] = None) -> bool: + return self.add_edges([(from_id, to_id, edge_type, metadata)]) != 0 + + def replace_edge(self, from_id: int, to_id: int, edge_type: str, + metadata: Optional[bytes] = None) -> None: + encoded_type = self._edge_type_bytes(edge_type) + if metadata is not None and not isinstance(metadata, bytes): + raise TypeError("edge metadata must be bytes or None") + with self._lock: + buffer = ctypes.create_string_buffer(metadata) if metadata else None + pointer = ctypes.cast(buffer, ctypes.c_void_p) if buffer else None + if not _lib.qihse_vector_db_replace_edge( + self._ptr, from_id, to_id, encoded_type, pointer, + len(metadata) if metadata else 0, + ): + raise RuntimeError("failed to replace edge") + + def remove_edge(self, from_id: int, to_id: int, edge_type: str) -> None: + encoded_type = self._edge_type_bytes(edge_type) + with self._lock: + if not _lib.qihse_vector_db_remove_edge(self._ptr, from_id, to_id, encoded_type): + raise RuntimeError("failed to remove edge") + + def neighbors(self, node_id: int, edge_type: Optional[str] = None, + direction: int = 0, limit: int = 1024) -> list[int]: + if direction not in (0, 1, 2) or limit <= 0: + raise ValueError("direction must be 0, 1, or 2 and limit must be positive") + encoded_type = self._edge_type_bytes(edge_type) if edge_type is not None else None + with self._lock: + output = (ctypes.c_uint64 * limit)() + count = _lib.qihse_vector_db_get_typed_neighbors( + self._ptr, node_id, encoded_type, direction, output, limit + ) + if count < 0: + raise RuntimeError("failed to retrieve neighbors") + return list(output[:count]) + + def edge_records(self, node_id: int, edge_type: Optional[str] = None, + direction: int = 0, limit: int = 1024) -> list[tuple[int, int, str, bytes]]: + if direction not in (0, 1, 2) or limit <= 0: + raise ValueError("direction must be 0, 1, or 2 and limit must be positive") + encoded_type = self._edge_type_bytes(edge_type) if edge_type is not None else None + with self._lock: + output = (CEdgeResult * limit)() + count = _lib.qihse_vector_db_get_edge_records( + self._ptr, node_id, encoded_type, direction, output, limit + ) + if count < 0: + raise RuntimeError("failed to retrieve edge records") + try: + return [ + (output[index].from_id, output[index].to_id, + bytes(output[index].edge_type).split(b"\x00", 1)[0].decode("utf-8"), + ctypes.string_at(output[index].metadata, output[index].metadata_size) + if output[index].metadata_size else b"") + for index in range(count) + ] + finally: + _lib.qihse_vector_db_free_edge_records(output, count) + def build_graph(self, M: int = 16, ef_construction: int = 200) -> None: """Build the graph index sidecar.""" with self._lock: diff --git a/src/broad_oak/qihse_vector_db.c b/src/broad_oak/qihse_vector_db.c index ca345f2..dd45ba4 100644 --- a/src/broad_oak/qihse_vector_db.c +++ b/src/broad_oak/qihse_vector_db.c @@ -31,6 +31,7 @@ static inline int munmap(void *addr, size_t length) { return -1; } #include #include #include +#include #include #include #include @@ -67,7 +68,23 @@ static inline int munmap(void *addr, size_t length) { return -1; } #define QIHSE_VDB_WAL_DELETE 3u #define QIHSE_VDB_WAL_UPDATE 4u #define QIHSE_VDB_WAL_UPSERT 5u +#define QIHSE_VDB_WAL_EDGE_ADD 6u +#define QIHSE_VDB_WAL_EDGE_REPLACE 7u +#define QIHSE_VDB_WAL_EDGE_REMOVE 8u #define QIHSE_VDB_WAL_NO_PREV UINT64_MAX +#define QIHSE_VDB_EDGE_MAGIC "QIHSEEDG" +#define QIHSE_VDB_EDGE_VERSION 1u +#define QIHSE_VDB_EDGE_HEADER_SIZE 72u +#define QIHSE_VDB_EDGE_SOURCE_SIZE 24u +#define QIHSE_VDB_EDGE_RECORD_SIZE 40u + +typedef struct qihse_vdb_edge_s { + uint64_t from_id; + uint64_t to_id; + char edge_type[QIHSE_EDGE_TYPE_MAX + 1u]; + void* metadata; + size_t metadata_size; +} qihse_vdb_edge_t; #define QIHSE_VDB_MAGNITUDE_ROW_BYTES 1u #define QIHSE_VDB_TRINARY_NEUTRAL_TRYTE 121u #define QIHSE_VDB_SCALAR_CANDIDATE_MULTIPLIER 12u @@ -273,15 +290,12 @@ struct qihse_vector_db_s { qihse_memory_superposition_state_t superposition_state; /* Explicit graph edge table (QQL/Graph DB) */ - struct { - uint64_t from_id; - uint64_t to_id; - char edge_type[32]; - void* metadata; - size_t metadata_size; - } * explicit_edges; + qihse_vdb_edge_t* explicit_edges; size_t explicit_edge_count; size_t explicit_edge_capacity; + bool explicit_edges_dirty; + pthread_mutex_t explicit_edge_mutex; + bool explicit_edge_mutex_initialized; }; typedef struct qihse_vdb_wal_add_s { @@ -372,6 +386,390 @@ static void qihse_vdb_int8_destroy(qihse_vector_db_t vdb); static float qihse_vdb_euclidean_distance(const float* a, const float* b, size_t n); static const float* qihse_hnsw_vdb_get_vector(void* ctx, uint32_t node_id); static const float* qihse_vdb_vector_at(qihse_vector_db_t vdb, const qihse_index_row_t* row); +static bool qihse_vdb_id_exists(const qihse_vector_db_t vdb, uint64_t id); + +static int qihse_vdb_edge_compare(const void* lhs, const void* rhs) { + const qihse_vdb_edge_t* a = (const qihse_vdb_edge_t*)lhs; + const qihse_vdb_edge_t* b = (const qihse_vdb_edge_t*)rhs; + int type_cmp; + if (a->from_id < b->from_id) return -1; + if (a->from_id > b->from_id) return 1; + type_cmp = strcmp(a->edge_type, b->edge_type); + if (type_cmp != 0) return type_cmp; + if (a->to_id < b->to_id) return -1; + if (a->to_id > b->to_id) return 1; + return 0; +} + +static void qihse_vdb_edge_array_free(qihse_vdb_edge_t* edges, size_t count) { + size_t i; + if (!edges) return; + for (i = 0u; i < count; i++) free(edges[i].metadata); + free(edges); +} + +static bool qihse_vdb_edge_type_valid(const char* edge_type) { + size_t len; + if (!edge_type) { + errno = EINVAL; + return false; + } + len = strnlen(edge_type, QIHSE_EDGE_TYPE_MAX + 2u); + if (len == 0u || len > QIHSE_EDGE_TYPE_MAX) { + errno = EINVAL; + return false; + } + return true; +} + +static bool qihse_vdb_edge_endpoints_valid(qihse_vector_db_t vdb, + uint64_t from_id, + uint64_t to_id) { + if (!qihse_vdb_id_exists(vdb, from_id) || !qihse_vdb_id_exists(vdb, to_id)) { + errno = ENOENT; + return false; + } + return true; +} + +static bool qihse_vdb_edge_copy(qihse_vdb_edge_t* dst, + const qihse_vdb_edge_t* src) { + *dst = *src; + dst->metadata = NULL; + if (src->metadata_size != 0u) { + dst->metadata = malloc(src->metadata_size); + if (!dst->metadata) { + errno = ENOMEM; + return false; + } + memcpy(dst->metadata, src->metadata, src->metadata_size); + } + return true; +} + +static bool qihse_vdb_edge_from_input(qihse_vdb_edge_t* dst, + const qihse_edge_input_t* src) { + size_t type_len; + if (!src || !qihse_vdb_edge_type_valid(src->edge_type) || + (src->metadata_size != 0u && !src->metadata)) { + errno = EINVAL; + return false; + } + memset(dst, 0, sizeof(*dst)); + dst->from_id = src->from_id; + dst->to_id = src->to_id; + type_len = strlen(src->edge_type); + memcpy(dst->edge_type, src->edge_type, type_len + 1u); + if (src->metadata_size != 0u) { + dst->metadata = malloc(src->metadata_size); + if (!dst->metadata) { + errno = ENOMEM; + return false; + } + memcpy(dst->metadata, src->metadata, src->metadata_size); + dst->metadata_size = src->metadata_size; + } + return true; +} + +static ssize_t qihse_vdb_edge_find(const qihse_vdb_edge_t* edges, + size_t count, + uint64_t from_id, + uint64_t to_id, + const char* edge_type) { + size_t i; + for (i = 0u; i < count; i++) { + if (edges[i].from_id == from_id && edges[i].to_id == to_id && + strcmp(edges[i].edge_type, edge_type) == 0) return (ssize_t)i; + } + return -1; +} + +static bool qihse_vdb_edge_metadata_equal(const qihse_vdb_edge_t* edge, + const void* metadata, + size_t metadata_size) { + return edge->metadata_size == metadata_size && + (metadata_size == 0u || memcmp(edge->metadata, metadata, metadata_size) == 0); +} + +static bool qihse_vdb_stage_edge_mutation(qihse_vector_db_t vdb, + uint32_t op, + const qihse_edge_input_t* inputs, + size_t input_count, + qihse_vdb_edge_t** out_edges, + size_t* out_count, + size_t* changed_count) { + qihse_vdb_edge_t* staged; + size_t capacity; + size_t count = vdb->explicit_edge_count; + size_t changed = 0u; + size_t i; + + if (!inputs || input_count == 0u || !out_edges || !out_count) { + errno = EINVAL; + return false; + } + if (input_count > SIZE_MAX - count) { + errno = EOVERFLOW; + return false; + } + capacity = count + input_count; + staged = (qihse_vdb_edge_t*)calloc(capacity ? capacity : 1u, sizeof(*staged)); + if (!staged) { + errno = ENOMEM; + return false; + } + for (i = 0u; i < count; i++) { + if (!qihse_vdb_edge_copy(&staged[i], &vdb->explicit_edges[i])) goto fail; + } + for (i = 0u; i < input_count; i++) { + ssize_t found; + if (!qihse_vdb_edge_type_valid(inputs[i].edge_type) || + (inputs[i].metadata_size != 0u && !inputs[i].metadata) || + !qihse_vdb_edge_endpoints_valid(vdb, inputs[i].from_id, inputs[i].to_id)) { + goto fail; + } + found = qihse_vdb_edge_find(staged, count, inputs[i].from_id, + inputs[i].to_id, inputs[i].edge_type); + if (op == QIHSE_VDB_WAL_EDGE_REMOVE) { + if (found >= 0) { + free(staged[found].metadata); + if ((size_t)found + 1u < count) { + memmove(&staged[found], &staged[found + 1u], + (count - (size_t)found - 1u) * sizeof(*staged)); + } + count--; + memset(&staged[count], 0, sizeof(*staged)); + changed++; + } + } else if (found >= 0) { + if (op == QIHSE_VDB_WAL_EDGE_REPLACE && + !qihse_vdb_edge_metadata_equal(&staged[found], inputs[i].metadata, + inputs[i].metadata_size)) { + void* replacement = NULL; + if (inputs[i].metadata_size != 0u) { + replacement = malloc(inputs[i].metadata_size); + if (!replacement) { errno = ENOMEM; goto fail; } + memcpy(replacement, inputs[i].metadata, inputs[i].metadata_size); + } + free(staged[found].metadata); + staged[found].metadata = replacement; + staged[found].metadata_size = inputs[i].metadata_size; + changed++; + } + } else { + if (op == QIHSE_VDB_WAL_EDGE_REPLACE) { + errno = ENOENT; + goto fail; + } + if (!qihse_vdb_edge_from_input(&staged[count], &inputs[i])) goto fail; + count++; + changed++; + } + } + qsort(staged, count, sizeof(*staged), qihse_vdb_edge_compare); + *out_edges = staged; + *out_count = count; + if (changed_count) *changed_count = changed; + return true; +fail: + qihse_vdb_edge_array_free(staged, count); + return false; +} + +static bool qihse_vdb_encode_edges(qihse_vector_db_t vdb, + uint8_t** out, + size_t* out_size) { + size_t source_count = 0u; + size_t string_bytes = 0u; + size_t source_bytes; + size_t record_bytes; + size_t total; + size_t i; + uint8_t* data; + size_t data_offset; + size_t source_index = 0u; + + if (!vdb || !out || !out_size) { errno = EINVAL; return false; } + *out = NULL; + *out_size = 0u; + for (i = 0u; i < vdb->explicit_edge_count; i++) { + if (i == 0u || vdb->explicit_edges[i].from_id != vdb->explicit_edges[i - 1u].from_id) + source_count++; + if (!qihse_checked_add_size(string_bytes, strlen(vdb->explicit_edges[i].edge_type), + &string_bytes) || + !qihse_checked_add_size(string_bytes, vdb->explicit_edges[i].metadata_size, + &string_bytes)) return false; + } + if (!qihse_checked_mul_size(source_count, QIHSE_VDB_EDGE_SOURCE_SIZE, &source_bytes) || + !qihse_checked_mul_size(vdb->explicit_edge_count, QIHSE_VDB_EDGE_RECORD_SIZE, + &record_bytes) || + !qihse_checked_add_size(QIHSE_VDB_EDGE_HEADER_SIZE, source_bytes, &total) || + !qihse_checked_add_size(total, record_bytes, &total) || + !qihse_checked_add_size(total, string_bytes, &total)) return false; + data = (uint8_t*)calloc(total ? total : 1u, 1u); + if (!data) { errno = ENOMEM; return false; } + memcpy(data, QIHSE_VDB_EDGE_MAGIC, 8u); + qihse_le_write_u32(data + 8u, QIHSE_VDB_EDGE_VERSION); + qihse_le_write_u32(data + 12u, QIHSE_VDB_EDGE_HEADER_SIZE); + qihse_le_write_u64(data + 16u, vdb->next_generation ? vdb->next_generation - 1u : 0u); + qihse_le_write_u64(data + 24u, (uint64_t)source_count); + qihse_le_write_u64(data + 32u, (uint64_t)vdb->explicit_edge_count); + qihse_le_write_u64(data + 40u, QIHSE_VDB_EDGE_HEADER_SIZE); + qihse_le_write_u64(data + 48u, QIHSE_VDB_EDGE_HEADER_SIZE + (uint64_t)source_bytes); + qihse_le_write_u64(data + 56u, QIHSE_VDB_EDGE_HEADER_SIZE + (uint64_t)source_bytes + + (uint64_t)record_bytes); + data_offset = QIHSE_VDB_EDGE_HEADER_SIZE + source_bytes + record_bytes; + for (i = 0u; i < vdb->explicit_edge_count; i++) { + qihse_vdb_edge_t* edge = &vdb->explicit_edges[i]; + uint8_t* record = data + QIHSE_VDB_EDGE_HEADER_SIZE + source_bytes + + i * QIHSE_VDB_EDGE_RECORD_SIZE; + size_t type_len = strlen(edge->edge_type); + if (i == 0u || edge->from_id != vdb->explicit_edges[i - 1u].from_id) { + size_t end = i + 1u; + uint8_t* source = data + QIHSE_VDB_EDGE_HEADER_SIZE + + source_index * QIHSE_VDB_EDGE_SOURCE_SIZE; + while (end < vdb->explicit_edge_count && + vdb->explicit_edges[end].from_id == edge->from_id) end++; + qihse_le_write_u64(source + 0u, edge->from_id); + qihse_le_write_u64(source + 8u, (uint64_t)i); + qihse_le_write_u64(source + 16u, (uint64_t)(end - i)); + source_index++; + } + qihse_le_write_u64(record + 0u, edge->to_id); + qihse_le_write_u64(record + 8u, (uint64_t)data_offset); + qihse_le_write_u32(record + 16u, (uint32_t)type_len); + qihse_le_write_u64(record + 24u, (uint64_t)(data_offset + type_len)); + qihse_le_write_u64(record + 32u, (uint64_t)edge->metadata_size); + memcpy(data + data_offset, edge->edge_type, type_len); + data_offset += type_len; + if (edge->metadata_size != 0u) { + memcpy(data + data_offset, edge->metadata, edge->metadata_size); + data_offset += edge->metadata_size; + } + } + qihse_le_write_u64(data + 64u, + qihse_fnv1a64(data + QIHSE_VDB_EDGE_HEADER_SIZE, + total - QIHSE_VDB_EDGE_HEADER_SIZE)); + *out = data; + *out_size = total; + return true; +} + +static bool qihse_vdb_load_edges(qihse_vector_db_t vdb) { + qihse_container_t ctr; + uint8_t* data = NULL; + size_t size = 0u; + uint64_t edge_count64 = 0u; + uint64_t source_count64 = 0u; + uint64_t source_offset = 0u; + uint64_t record_offset = 0u; + uint64_t data_offset = 0u; + qihse_vdb_edge_t* edges = NULL; + size_t i; + uint64_t expected_first = 0u; + bool ok = false; + + if (!vdb || !vdb->db_path) { errno = EINVAL; return false; } + if (!qihse_ctr_open_read(vdb->db_path, &ctr)) return false; + if (!qihse_ctr_find_section(&ctr, QIHSE_CTR_SEC_EDGES)) { + qihse_ctr_close(&ctr); + return true; + } + if (!qihse_ctr_read_section_alloc(&ctr, QIHSE_CTR_SEC_EDGES, &data, &size)) { + qihse_ctr_close(&ctr); + return false; + } + qihse_ctr_close(&ctr); + if (size < QIHSE_VDB_EDGE_HEADER_SIZE || + memcmp(data, QIHSE_VDB_EDGE_MAGIC, 8u) != 0 || + qihse_le_read_u32(data + 8u) != QIHSE_VDB_EDGE_VERSION || + qihse_le_read_u32(data + 12u) != QIHSE_VDB_EDGE_HEADER_SIZE || + qihse_fnv1a64(data + QIHSE_VDB_EDGE_HEADER_SIZE, + size - QIHSE_VDB_EDGE_HEADER_SIZE) != qihse_le_read_u64(data + 64u)) { + errno = EINVAL; + goto done; + } + source_count64 = qihse_le_read_u64(data + 24u); + edge_count64 = qihse_le_read_u64(data + 32u); + source_offset = qihse_le_read_u64(data + 40u); + record_offset = qihse_le_read_u64(data + 48u); + data_offset = qihse_le_read_u64(data + 56u); + if (source_count64 > SIZE_MAX || edge_count64 > SIZE_MAX || + source_offset != QIHSE_VDB_EDGE_HEADER_SIZE || + source_count64 > (size - (size_t)source_offset) / QIHSE_VDB_EDGE_SOURCE_SIZE || + record_offset != source_offset + source_count64 * QIHSE_VDB_EDGE_SOURCE_SIZE || + edge_count64 > (size - (size_t)record_offset) / QIHSE_VDB_EDGE_RECORD_SIZE || + data_offset != record_offset + edge_count64 * QIHSE_VDB_EDGE_RECORD_SIZE || + data_offset > size) { + errno = EINVAL; + goto done; + } + edges = (qihse_vdb_edge_t*)calloc(edge_count64 ? (size_t)edge_count64 : 1u, + sizeof(*edges)); + if (!edges) { errno = ENOMEM; goto done; } + for (i = 0u; i < (size_t)source_count64; i++) { + const uint8_t* source = data + source_offset + i * QIHSE_VDB_EDGE_SOURCE_SIZE; + uint64_t from_id = qihse_le_read_u64(source + 0u); + uint64_t first = qihse_le_read_u64(source + 8u); + uint64_t count = qihse_le_read_u64(source + 16u); + size_t j; + if (count == 0u || first != expected_first || first > edge_count64 || + count > edge_count64 - first || + (i != 0u && from_id <= qihse_le_read_u64( + data + source_offset + (i - 1u) * QIHSE_VDB_EDGE_SOURCE_SIZE))) { + errno = EINVAL; + goto done; + } + expected_first = first + count; + for (j = (size_t)first; j < (size_t)(first + count); j++) { + const uint8_t* record = data + record_offset + j * QIHSE_VDB_EDGE_RECORD_SIZE; + uint64_t type_off = qihse_le_read_u64(record + 8u); + uint32_t type_len = qihse_le_read_u32(record + 16u); + uint64_t meta_off = qihse_le_read_u64(record + 24u); + uint64_t meta_len = qihse_le_read_u64(record + 32u); + if (type_len == 0u || type_len > QIHSE_EDGE_TYPE_MAX || type_off < data_offset || + type_off > size || type_len > size - type_off || meta_off < data_offset || + meta_off > size || meta_len > size - meta_off || meta_len > SIZE_MAX) { + errno = EINVAL; + goto done; + } + edges[j].from_id = from_id; + edges[j].to_id = qihse_le_read_u64(record + 0u); + memcpy(edges[j].edge_type, data + type_off, type_len); + edges[j].edge_type[type_len] = '\0'; + if (meta_len != 0u) { + edges[j].metadata = malloc((size_t)meta_len); + if (!edges[j].metadata) { errno = ENOMEM; goto done; } + memcpy(edges[j].metadata, data + meta_off, (size_t)meta_len); + edges[j].metadata_size = (size_t)meta_len; + } + } + } + if (expected_first != edge_count64 || + ((edge_count64 == 0u) != (source_count64 == 0u))) { + errno = EINVAL; + goto done; + } + if ((size_t)edge_count64 != 0u) { + for (i = 0u; i < (size_t)edge_count64; i++) { + if (!qihse_vdb_edge_endpoints_valid(vdb, edges[i].from_id, edges[i].to_id) || + (i != 0u && qihse_vdb_edge_compare(&edges[i - 1u], &edges[i]) >= 0)) { + errno = EINVAL; + goto done; + } + } + } + vdb->explicit_edges = edges; + vdb->explicit_edge_count = (size_t)edge_count64; + vdb->explicit_edge_capacity = (size_t)edge_count64; + edges = NULL; + ok = true; +done: + qihse_vdb_edge_array_free(edges, (size_t)(edge_count64 > SIZE_MAX ? 0u : edge_count64)); + free(data); + return ok; +} /* ============================================================================ * GRAPH SIDECAR PERSISTENCE @@ -3312,6 +3710,177 @@ static bool qihse_vdb_write_wal_add(qihse_vector_db_t vdb, const qihse_vdb_wal_a return qihse_vdb_write_wal_vectors(vdb, QIHSE_VDB_WAL_ADD, add); } +static bool qihse_vdb_write_edge_wal(qihse_vector_db_t vdb, + uint32_t type, + uint64_t generation, + const qihse_edge_input_t* edges, + size_t edge_count) { + uint8_t* payload = NULL; + size_t payload_size = sizeof(uint64_t); + size_t offset = 0u; + size_t i; + uint8_t mut_hdr[QIHSE_VDB_WAL_HEADER_SIZE]; + uint8_t cmt_hdr[QIHSE_VDB_WAL_HEADER_SIZE]; + uint8_t commit_payload[16]; + uint64_t mut_crc; + uint64_t cmt_crc; + uint64_t mut_off; + uint64_t cmt_off; + size_t total_size; + uint8_t* wal_buf = NULL; + qihse_container_t ctr; + bool ok; + + if (!vdb || !vdb->db_path || !edges || edge_count == 0u || + (type != QIHSE_VDB_WAL_EDGE_ADD && type != QIHSE_VDB_WAL_EDGE_REPLACE && + type != QIHSE_VDB_WAL_EDGE_REMOVE)) { + errno = EINVAL; + return false; + } + for (i = 0u; i < edge_count; i++) { + size_t type_len; + if (!qihse_vdb_edge_type_valid(edges[i].edge_type) || + (edges[i].metadata_size != 0u && !edges[i].metadata)) return false; + type_len = strlen(edges[i].edge_type); + if (!qihse_checked_add_size(payload_size, 4u * sizeof(uint64_t), &payload_size) || + !qihse_checked_add_size(payload_size, type_len, &payload_size) || + !qihse_checked_add_size(payload_size, edges[i].metadata_size, &payload_size)) + return false; + } + payload = (uint8_t*)malloc(payload_size); + if (!payload) { errno = ENOMEM; return false; } + qihse_le_write_u64(payload, (uint64_t)edge_count); + offset = sizeof(uint64_t); + for (i = 0u; i < edge_count; i++) { + size_t type_len = strlen(edges[i].edge_type); + qihse_le_write_u64(payload + offset + 0u, edges[i].from_id); + qihse_le_write_u64(payload + offset + 8u, edges[i].to_id); + qihse_le_write_u64(payload + offset + 16u, (uint64_t)type_len); + qihse_le_write_u64(payload + offset + 24u, (uint64_t)edges[i].metadata_size); + offset += 4u * sizeof(uint64_t); + memcpy(payload + offset, edges[i].edge_type, type_len); + offset += type_len; + if (edges[i].metadata_size != 0u) { + memcpy(payload + offset, edges[i].metadata, edges[i].metadata_size); + offset += edges[i].metadata_size; + } + } + mut_crc = qihse_fnv1a64(payload, payload_size); + memset(mut_hdr, 0, sizeof(mut_hdr)); + memcpy(mut_hdr, QIHSE_VDB_WAL_MAGIC, 8u); + qihse_le_write_u32(mut_hdr + 8u, QIHSE_VDB_WAL_VERSION); + qihse_le_write_u32(mut_hdr + 12u, type); + qihse_le_write_u64(mut_hdr + 16u, generation); + qihse_le_write_u64(mut_hdr + 24u, (uint64_t)payload_size); + qihse_le_write_u64(mut_hdr + 32u, mut_crc); + qihse_le_write_u64(mut_hdr + 40u, vdb->wal_last_record_offset); + mut_off = vdb->wal_bytes_pending; + cmt_off = mut_off + QIHSE_VDB_WAL_HEADER_SIZE + (uint64_t)payload_size; + qihse_le_write_u64(commit_payload + 0u, mut_off); + qihse_le_write_u64(commit_payload + 8u, mut_crc); + cmt_crc = qihse_fnv1a64(commit_payload, sizeof(commit_payload)); + memset(cmt_hdr, 0, sizeof(cmt_hdr)); + memcpy(cmt_hdr, QIHSE_VDB_WAL_MAGIC, 8u); + qihse_le_write_u32(cmt_hdr + 8u, QIHSE_VDB_WAL_VERSION); + qihse_le_write_u32(cmt_hdr + 12u, QIHSE_VDB_WAL_COMMIT); + qihse_le_write_u64(cmt_hdr + 16u, generation); + qihse_le_write_u64(cmt_hdr + 24u, sizeof(commit_payload)); + qihse_le_write_u64(cmt_hdr + 32u, cmt_crc); + qihse_le_write_u64(cmt_hdr + 40u, mut_off); + if (!qihse_checked_add_size(2u * QIHSE_VDB_WAL_HEADER_SIZE, payload_size, + &total_size) || + !qihse_checked_add_size(total_size, sizeof(commit_payload), &total_size)) { + free(payload); + return false; + } + wal_buf = (uint8_t*)malloc(total_size); + if (!wal_buf) { free(payload); errno = ENOMEM; return false; } + offset = 0u; + memcpy(wal_buf + offset, mut_hdr, sizeof(mut_hdr)); offset += sizeof(mut_hdr); + memcpy(wal_buf + offset, payload, payload_size); offset += payload_size; + memcpy(wal_buf + offset, cmt_hdr, sizeof(cmt_hdr)); offset += sizeof(cmt_hdr); + memcpy(wal_buf + offset, commit_payload, sizeof(commit_payload)); + free(payload); + ok = qihse_ctr_open_write(vdb->db_path, false, &ctr); + if (ok) { + ok = qihse_ctr_wal_append(&ctr, wal_buf, total_size) && qihse_ctr_fsync(&ctr); + qihse_ctr_close(&ctr); + } + free(wal_buf); + if (ok) { + vdb->wal_bytes_pending += (uint64_t)total_size; + vdb->wal_last_record_offset = cmt_off; + } + return ok; +} + +static bool qihse_vdb_apply_edge_wal_payload(qihse_vector_db_t vdb, + uint32_t type, + uint64_t generation, + const uint8_t* payload, + size_t payload_size) { + qihse_edge_input_t* inputs = NULL; + size_t count; + size_t offset = sizeof(uint64_t); + size_t i; + qihse_vdb_edge_t* staged = NULL; + size_t staged_count = 0u; + size_t changed = 0u; + bool ok = false; + + if (!payload || payload_size < sizeof(uint64_t)) { errno = EINVAL; return false; } + if (!qihse_vdb_u64_to_size(qihse_le_read_u64(payload), &count) || count == 0u) return false; + inputs = (qihse_edge_input_t*)calloc(count, sizeof(*inputs)); + if (!inputs) { errno = ENOMEM; return false; } + for (i = 0u; i < count; i++) { + uint64_t type_len64; + uint64_t meta_len64; + size_t type_len; + size_t meta_len; + char* type_copy; + if (payload_size - offset < 4u * sizeof(uint64_t)) { errno = EINVAL; goto done; } + inputs[i].from_id = qihse_le_read_u64(payload + offset + 0u); + inputs[i].to_id = qihse_le_read_u64(payload + offset + 8u); + type_len64 = qihse_le_read_u64(payload + offset + 16u); + meta_len64 = qihse_le_read_u64(payload + offset + 24u); + offset += 4u * sizeof(uint64_t); + if (!qihse_vdb_u64_to_size(type_len64, &type_len) || + !qihse_vdb_u64_to_size(meta_len64, &meta_len) || + type_len == 0u || type_len > QIHSE_EDGE_TYPE_MAX || + type_len > payload_size - offset) { errno = EINVAL; goto done; } + type_copy = (char*)malloc(type_len + 1u); + if (!type_copy) { errno = ENOMEM; goto done; } + memcpy(type_copy, payload + offset, type_len); + type_copy[type_len] = '\0'; + inputs[i].edge_type = type_copy; + offset += type_len; + if (meta_len > payload_size - offset) { errno = EINVAL; goto done; } + inputs[i].metadata = meta_len ? payload + offset : NULL; + inputs[i].metadata_size = meta_len; + offset += meta_len; + } + if (offset != payload_size || + !qihse_vdb_stage_edge_mutation(vdb, type, inputs, count, + &staged, &staged_count, &changed)) goto done; + qihse_vdb_edge_array_free(vdb->explicit_edges, vdb->explicit_edge_count); + vdb->explicit_edges = staged; + vdb->explicit_edge_count = staged_count; + vdb->explicit_edge_capacity = staged_count; + vdb->explicit_edges_dirty = changed != 0u; + staged = NULL; + if (generation >= vdb->next_generation) vdb->next_generation = generation + 1u; + if (changed != 0u) vdb->dirty = true; + vdb->wal_records_replayed++; + ok = true; +done: + if (inputs) { + for (i = 0u; i < count; i++) free((void*)inputs[i].edge_type); + } + free(inputs); + qihse_vdb_edge_array_free(staged, staged_count); + return ok; +} + static bool qihse_vdb_apply_delete_payload(qihse_vector_db_t vdb, const qihse_vdb_wal_vectors_t* record) { size_t deleted = 0u; @@ -3460,6 +4029,12 @@ static bool qihse_vdb_replay_wal_payload(qihse_vector_db_t vdb, size_t metadata_total = 0u; bool ok = false; + if (type == QIHSE_VDB_WAL_EDGE_ADD || type == QIHSE_VDB_WAL_EDGE_REPLACE || + type == QIHSE_VDB_WAL_EDGE_REMOVE) { + return qihse_vdb_apply_edge_wal_payload(vdb, type, generation, + payload, payload_size); + } + if (!payload || payload_size < 32u) { errno = EINVAL; return false; @@ -3621,7 +4196,9 @@ static bool qihse_vdb_replay_wal(qihse_vector_db_t vdb) { if (version != QIHSE_VDB_WAL_VERSION || (type != QIHSE_VDB_WAL_ADD && type != QIHSE_VDB_WAL_COMMIT && type != QIHSE_VDB_WAL_DELETE && type != QIHSE_VDB_WAL_UPDATE && - type != QIHSE_VDB_WAL_UPSERT) || + type != QIHSE_VDB_WAL_UPSERT && type != QIHSE_VDB_WAL_EDGE_ADD && + type != QIHSE_VDB_WAL_EDGE_REPLACE && + type != QIHSE_VDB_WAL_EDGE_REMOVE) || payload_size64 > (uint64_t)SIZE_MAX || (record_start != 0u && prev_offset_val != last_record_offset) || (record_start == 0u && prev_offset_val != QIHSE_VDB_WAL_NO_PREV && @@ -3970,6 +4547,12 @@ qihse_vector_db_t qihse_vector_db_open( vdb->wal_last_record_offset = QIHSE_VDB_WAL_NO_PREV; vdb->trinary_status = QIHSE_VDB_TRINARY_ABSENT; vdb->magnitude_status = QIHSE_VDB_MAGNITUDE_ABSENT; + if (pthread_mutex_init(&vdb->explicit_edge_mutex, NULL) != 0) { + free(vdb); + errno = ENOMEM; + return NULL; + } + vdb->explicit_edge_mutex_initialized = true; /* Hierarchical storage defaults */ vdb->memory_hot_threshold = 100.0; /* 100 accesses per evaluation window = hot */ @@ -4015,6 +4598,10 @@ qihse_vector_db_t qihse_vector_db_open( return NULL; } loaded = true; + if (!qihse_vdb_load_edges(vdb)) { + qihse_vector_db_destroy(vdb); + return NULL; + } } else if (!create && !has_wal) { printf("DEBUG: qihse_vector_db_open failed at %d\n", __LINE__); qihse_vector_db_destroy(vdb); @@ -4302,6 +4889,19 @@ bool qihse_vector_db_delete_by_ids( } return false; } + pthread_mutex_lock(&vdb->explicit_edge_mutex); + for (i = 0u; i < vdb->explicit_edge_count; i++) { + size_t id_index; + for (id_index = 0u; id_index < count; id_index++) { + if (vdb->explicit_edges[i].from_id == vector_ids[id_index] || + vdb->explicit_edges[i].to_id == vector_ids[id_index]) { + pthread_mutex_unlock(&vdb->explicit_edge_mutex); + errno = EBUSY; + return false; + } + } + } + pthread_mutex_unlock(&vdb->explicit_edge_mutex); for (i = 0u; i < count; i++) { size_t row_index = 0u; bool found; @@ -5972,8 +6572,11 @@ bool qihse_vector_db_flush(qihse_vector_db_t vdb) { qihse_vector_store_flush_t flush; uint8_t* trinary = NULL; uint8_t* magnitude = NULL; + uint8_t* explicit_edges = NULL; size_t trinary_bytes = 0u; size_t magnitude_bytes = 0u; + size_t explicit_edges_bytes = 0u; + uint64_t explicit_edge_generation; bool ok; if (!vdb) { @@ -6000,6 +6603,15 @@ bool qihse_vector_db_flush(qihse_vector_db_t vdb) { return false; } } + pthread_mutex_lock(&vdb->explicit_edge_mutex); + explicit_edge_generation = vdb->next_generation ? vdb->next_generation - 1u : 0u; + ok = qihse_vdb_encode_edges(vdb, &explicit_edges, &explicit_edges_bytes); + pthread_mutex_unlock(&vdb->explicit_edge_mutex); + if (!ok) { + free(trinary); + free(magnitude); + return false; + } memset(&flush, 0, sizeof(flush)); flush.vector_dims = (uint32_t)vdb->vector_dims; flush.commit_generation = vdb->next_generation ? vdb->next_generation - 1u : 0u; @@ -6021,6 +6633,8 @@ bool qihse_vector_db_flush(qihse_vector_db_t vdb) { flush.magnitude_generation = flush.commit_generation; flush.magnitude_row_bytes = vdb->magnitude_row_bytes; flush.magnitude_flags = QIHSE_VSTORE_MAG_PRESENT | QIHSE_VSTORE_MAG_VALID; + flush.explicit_edges = explicit_edges; + flush.explicit_edges_bytes = explicit_edges_bytes; ok = qihse_vector_store_flush(vdb->db_path, &flush); if (ok) { @@ -6040,6 +6654,14 @@ bool qihse_vector_db_flush(qihse_vector_db_t vdb) { vdb->idmap_dirty = false; qihse_vdb_tier_save(vdb); vdb->idmap_valid = true; + pthread_mutex_lock(&vdb->explicit_edge_mutex); + if ((vdb->next_generation ? vdb->next_generation - 1u : 0u) == + explicit_edge_generation) { + vdb->explicit_edges_dirty = false; + } else { + vdb->dirty = true; + } + pthread_mutex_unlock(&vdb->explicit_edge_mutex); if (trinary_bytes != 0u) { qihse_vdb_clear_trinary_cache(vdb); vdb->trinary = trinary; @@ -6063,6 +6685,7 @@ bool qihse_vector_db_flush(qihse_vector_db_t vdb) { } free(trinary); free(magnitude); + free(explicit_edges); return ok; } @@ -6522,11 +7145,9 @@ void qihse_vector_db_destroy(qihse_vector_db_t vdb) { qihse_vdb_sparse_index_destroy(vdb->sparse_index); vdb->sparse_index = NULL; /* Free explicit graph edge table */ - if (vdb->explicit_edges) { - for (size_t i = 0; i < vdb->explicit_edge_count; i++) { - free(vdb->explicit_edges[i].metadata); - } - free(vdb->explicit_edges); + qihse_vdb_edge_array_free(vdb->explicit_edges, vdb->explicit_edge_count); + if (vdb->explicit_edge_mutex_initialized) { + pthread_mutex_destroy(&vdb->explicit_edge_mutex); } free(vdb); } @@ -6752,38 +7373,130 @@ bool qihse_vector_db_add_edge( const void* metadata, size_t metadata_size ) { - if (!vdb || !edge_type) return false; - - /* Grow edge table if needed */ - if (vdb->explicit_edge_count >= vdb->explicit_edge_capacity) { - size_t new_cap = vdb->explicit_edge_capacity == 0 ? 64 : vdb->explicit_edge_capacity * 2; - void* new_edges = realloc(vdb->explicit_edges, new_cap * sizeof(*vdb->explicit_edges)); - if (!new_edges) return false; - vdb->explicit_edges = new_edges; - vdb->explicit_edge_capacity = new_cap; - } - - /* Store the edge */ - size_t idx = vdb->explicit_edge_count; - vdb->explicit_edges[idx].from_id = from_id; - vdb->explicit_edges[idx].to_id = to_id; - strncpy(vdb->explicit_edges[idx].edge_type, edge_type, sizeof(vdb->explicit_edges[idx].edge_type) - 1); - vdb->explicit_edges[idx].edge_type[sizeof(vdb->explicit_edges[idx].edge_type) - 1] = '\0'; - - /* Copy metadata if provided */ - if (metadata && metadata_size > 0) { - vdb->explicit_edges[idx].metadata = malloc(metadata_size); - if (!vdb->explicit_edges[idx].metadata) return false; - memcpy(vdb->explicit_edges[idx].metadata, metadata, metadata_size); - vdb->explicit_edges[idx].metadata_size = metadata_size; - } else { - vdb->explicit_edges[idx].metadata = NULL; - vdb->explicit_edges[idx].metadata_size = 0; - } + qihse_edge_input_t edge; + edge.from_id = from_id; + edge.to_id = to_id; + edge.edge_type = edge_type; + edge.metadata = metadata; + edge.metadata_size = metadata_size; + return qihse_vector_db_add_edges(vdb, &edge, 1u, NULL); +} + +static bool qihse_vdb_mutate_edges(qihse_vector_db_t vdb, + uint32_t op, + const qihse_edge_input_t* edges, + size_t edge_count, + size_t* changed_count) { + qihse_vdb_edge_t* staged = NULL; + size_t staged_count = 0u; + size_t changed = 0u; + uint64_t generation; + bool ok = false; - vdb->explicit_edge_count++; + if (changed_count) *changed_count = 0u; + if (!qihse_vdb_ensure_writable(vdb) || !edges || edge_count == 0u) { + if (edges == NULL || edge_count == 0u) errno = EINVAL; + return false; + } + pthread_mutex_lock(&vdb->explicit_edge_mutex); + if (!qihse_vdb_stage_edge_mutation(vdb, op, edges, edge_count, + &staged, &staged_count, &changed)) goto done; + if (changed == 0u) { + ok = true; + goto done; + } + generation = vdb->next_generation; + if (vdb->file_backed && + !qihse_vdb_write_edge_wal(vdb, op, generation, edges, edge_count)) goto done; + qihse_vdb_edge_array_free(vdb->explicit_edges, vdb->explicit_edge_count); + vdb->explicit_edges = staged; + vdb->explicit_edge_count = staged_count; + vdb->explicit_edge_capacity = staged_count; + staged = NULL; + vdb->next_generation = generation + 1u; vdb->dirty = true; - return true; + vdb->explicit_edges_dirty = true; + if (changed_count) *changed_count = changed; + ok = true; +done: + qihse_vdb_edge_array_free(staged, staged_count); + pthread_mutex_unlock(&vdb->explicit_edge_mutex); + return ok; +} + +bool qihse_vector_db_add_edges(qihse_vector_db_t vdb, + const qihse_edge_input_t* edges, + size_t edge_count, + size_t* changed_count) { + return qihse_vdb_mutate_edges(vdb, QIHSE_VDB_WAL_EDGE_ADD, edges, + edge_count, changed_count); +} + +bool qihse_vector_db_replace_edge(qihse_vector_db_t vdb, + uint64_t from_id, + uint64_t to_id, + const char* edge_type, + const void* metadata, + size_t metadata_size) { + qihse_edge_input_t edge; + edge.from_id = from_id; + edge.to_id = to_id; + edge.edge_type = edge_type; + edge.metadata = metadata; + edge.metadata_size = metadata_size; + return qihse_vdb_mutate_edges(vdb, QIHSE_VDB_WAL_EDGE_REPLACE, &edge, 1u, NULL); +} + +bool qihse_vector_db_remove_edge(qihse_vector_db_t vdb, + uint64_t from_id, + uint64_t to_id, + const char* edge_type) { + qihse_edge_input_t edge; + edge.from_id = from_id; + edge.to_id = to_id; + edge.edge_type = edge_type; + edge.metadata = NULL; + edge.metadata_size = 0u; + return qihse_vdb_mutate_edges(vdb, QIHSE_VDB_WAL_EDGE_REMOVE, &edge, 1u, NULL); +} + +static bool qihse_vdb_edge_matches(const qihse_vdb_edge_t* edge, + uint64_t node_id, + const char* edge_type, + qihse_edge_direction_t direction) { + bool direction_match = + (direction == QIHSE_EDGE_OUTGOING && edge->from_id == node_id) || + (direction == QIHSE_EDGE_INCOMING && edge->to_id == node_id) || + (direction == QIHSE_EDGE_BOTH && + (edge->from_id == node_id || edge->to_id == node_id)); + return direction_match && (!edge_type || strcmp(edge->edge_type, edge_type) == 0); +} + +int qihse_vector_db_get_typed_neighbors(qihse_vector_db_t vdb, + uint64_t node_id, + const char* edge_type, + qihse_edge_direction_t direction, + uint64_t* out_ids, + size_t max_edges) { + size_t found = 0u; + size_t i; + if (!vdb || !out_ids || max_edges == 0u || + direction < QIHSE_EDGE_OUTGOING || direction > QIHSE_EDGE_BOTH || + (edge_type && !qihse_vdb_edge_type_valid(edge_type))) { + errno = EINVAL; + return -1; + } + if (!qihse_vdb_id_exists(vdb, node_id)) { errno = ENOENT; return -1; } + pthread_mutex_lock(&vdb->explicit_edge_mutex); + for (i = 0u; i < vdb->explicit_edge_count && found < max_edges; i++) { + qihse_vdb_edge_t* edge = &vdb->explicit_edges[i]; + if (!qihse_vdb_edge_matches(edge, node_id, edge_type, direction)) continue; + out_ids[found++] = direction == QIHSE_EDGE_INCOMING ? edge->from_id : + (direction == QIHSE_EDGE_BOTH && edge->to_id == node_id ? + edge->from_id : edge->to_id); + } + pthread_mutex_unlock(&vdb->explicit_edge_mutex); + return (int)found; } int qihse_vector_db_get_edges( @@ -6793,17 +7506,61 @@ int qihse_vector_db_get_edges( uint64_t* out_ids, size_t max_edges ) { - if (!vdb || !out_ids || max_edges == 0) return -1; - - size_t found = 0; - for (size_t i = 0; i < vdb->explicit_edge_count && found < max_edges; i++) { - if (vdb->explicit_edges[i].from_id != from_id) continue; - if (edge_type && strcmp(vdb->explicit_edges[i].edge_type, edge_type) != 0) continue; - out_ids[found++] = vdb->explicit_edges[i].to_id; + return qihse_vector_db_get_typed_neighbors(vdb, from_id, edge_type, + QIHSE_EDGE_OUTGOING, + out_ids, max_edges); +} + +int qihse_vector_db_get_edge_records(qihse_vector_db_t vdb, + uint64_t node_id, + const char* edge_type, + qihse_edge_direction_t direction, + qihse_edge_result_t* results, + size_t max_edges) { + size_t found = 0u; + size_t i; + if (!vdb || !results || max_edges == 0u || + direction < QIHSE_EDGE_OUTGOING || direction > QIHSE_EDGE_BOTH || + (edge_type && !qihse_vdb_edge_type_valid(edge_type))) { + errno = EINVAL; + return -1; + } + if (!qihse_vdb_id_exists(vdb, node_id)) { errno = ENOENT; return -1; } + memset(results, 0, max_edges * sizeof(*results)); + pthread_mutex_lock(&vdb->explicit_edge_mutex); + for (i = 0u; i < vdb->explicit_edge_count && found < max_edges; i++) { + qihse_vdb_edge_t* edge = &vdb->explicit_edges[i]; + if (!qihse_vdb_edge_matches(edge, node_id, edge_type, direction)) continue; + results[found].from_id = edge->from_id; + results[found].to_id = edge->to_id; + memcpy(results[found].edge_type, edge->edge_type, sizeof(edge->edge_type)); + if (edge->metadata_size != 0u) { + results[found].metadata = malloc(edge->metadata_size); + if (!results[found].metadata) { + pthread_mutex_unlock(&vdb->explicit_edge_mutex); + qihse_vector_db_free_edge_records(results, found); + errno = ENOMEM; + return -1; + } + memcpy(results[found].metadata, edge->metadata, edge->metadata_size); + results[found].metadata_size = edge->metadata_size; + } + found++; } + pthread_mutex_unlock(&vdb->explicit_edge_mutex); return (int)found; } +void qihse_vector_db_free_edge_records(qihse_edge_result_t* results, size_t count) { + size_t i; + if (!results) return; + for (i = 0u; i < count; i++) { + free(results[i].metadata); + results[i].metadata = NULL; + results[i].metadata_size = 0u; + } +} + /* ============================================================================ * EMBEDDED QUERY EXECUTION (QQL & SQL) * ============================================================================ */ diff --git a/tests/qihse_edge_persistence_test.c b/tests/qihse_edge_persistence_test.c new file mode 100644 index 0000000..b1f8491 --- /dev/null +++ b/tests/qihse_edge_persistence_test.c @@ -0,0 +1,143 @@ +#include "qihse_vector_db.h" +#include "persistence/qihse_container.h" + +#include +#include +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + if (!(expr)) { \ + fprintf(stderr, "CHECK failed at %s:%d: %s (errno=%d)\n", \ + __FILE__, __LINE__, #expr, errno); \ + return 1; \ + } \ +} while (0) + +static int write_uncheckpointed_edges(const char* path) { + qihse_vector_db_t db; + const float vectors[] = {1.0f, 0.0f, 0.0f, 1.0f, 0.5f, 0.5f}; + const uint64_t ids[] = {101u, 102u, 103u}; + const char metadata[] = "callsite=alpha"; + + db = qihse_vector_db_open(QIHSE_VECTOR_DB_INMEMORY, NULL, path, + QIHSE_VDB_OPEN_CREATE | QIHSE_VDB_OPEN_FILE_BACKED | + QIHSE_VDB_OPEN_TRUNCATE); + CHECK(db != NULL); + CHECK(qihse_vector_db_add_vectors(db, vectors, 3u, 2u, ids, NULL, NULL)); + CHECK(qihse_vector_db_add_edge(db, 101u, 102u, "CALLS", + metadata, sizeof(metadata) - 1u)); + CHECK(qihse_vector_db_add_edge(db, 101u, 103u, "TESTED_BY", NULL, 0u)); + CHECK(!qihse_vector_db_delete_by_id(db, 102u)); + CHECK(errno == EBUSY); + _exit(0); +} + +static int verify_replay_and_checkpoint(const char* path) { + qihse_vector_db_t db; + uint64_t neighbors[4] = {0u}; + qihse_edge_result_t records[2]; + qihse_edge_input_t batch[2]; + size_t changed = 0u; + int count; + const char replacement[] = "callsite=beta"; + + db = qihse_vector_db_open(QIHSE_VECTOR_DB_INMEMORY, NULL, path, + QIHSE_VDB_OPEN_FILE_BACKED); + CHECK(db != NULL); + count = qihse_vector_db_get_edges(db, 101u, NULL, neighbors, 4u); + CHECK(count == 2); + CHECK(neighbors[0] == 102u && neighbors[1] == 103u); + + CHECK(qihse_vector_db_add_edge(db, 101u, 102u, "CALLS", "ignored", 7u)); + memset(records, 0, sizeof(records)); + count = qihse_vector_db_get_edge_records(db, 101u, "CALLS", + QIHSE_EDGE_OUTGOING, records, 2u); + CHECK(count == 1); + CHECK(records[0].metadata_size == strlen("callsite=alpha")); + CHECK(memcmp(records[0].metadata, "callsite=alpha", records[0].metadata_size) == 0); + qihse_vector_db_free_edge_records(records, 1u); + + CHECK(qihse_vector_db_replace_edge(db, 101u, 102u, "CALLS", + replacement, sizeof(replacement) - 1u)); + batch[0] = (qihse_edge_input_t){102u, 103u, "CALLS", NULL, 0u}; + batch[1] = (qihse_edge_input_t){103u, 101u, "IMPLEMENTS", "iface", 5u}; + CHECK(qihse_vector_db_add_edges(db, batch, 2u, &changed)); + CHECK(changed == 2u); + count = qihse_vector_db_get_typed_neighbors(db, 103u, NULL, + QIHSE_EDGE_INCOMING, + neighbors, 4u); + CHECK(count == 2); + CHECK(neighbors[0] == 102u || neighbors[1] == 102u); + CHECK(qihse_vector_db_checkpoint(db)); + CHECK(qihse_vector_db_close(db)); + + db = qihse_vector_db_open(QIHSE_VECTOR_DB_INMEMORY, NULL, path, + QIHSE_VDB_OPEN_FILE_BACKED | QIHSE_VDB_OPEN_READ_ONLY); + CHECK(db != NULL); + memset(records, 0, sizeof(records)); + count = qihse_vector_db_get_edge_records(db, 101u, "CALLS", + QIHSE_EDGE_OUTGOING, records, 2u); + CHECK(count == 1); + CHECK(records[0].metadata_size == sizeof(replacement) - 1u); + CHECK(memcmp(records[0].metadata, replacement, sizeof(replacement) - 1u) == 0); + qihse_vector_db_free_edge_records(records, 1u); + CHECK(!qihse_vector_db_remove_edge(db, 101u, 102u, "CALLS")); + CHECK(errno == EROFS); + qihse_vector_db_destroy(db); + return 0; +} + +static int verify_remove_and_old_container(const char* path) { + qihse_vector_db_t db; + qihse_container_t ctr; + qihse_ctr_section_buf_t remove_edges = {QIHSE_CTR_SEC_EDGES, NULL, 0u}; + uint64_t neighbors[2] = {0u}; + + db = qihse_vector_db_open(QIHSE_VECTOR_DB_INMEMORY, NULL, path, + QIHSE_VDB_OPEN_FILE_BACKED); + CHECK(db != NULL); + CHECK(qihse_vector_db_remove_edge(db, 101u, 102u, "CALLS")); + CHECK(qihse_vector_db_remove_edge(db, 101u, 102u, "CALLS")); + CHECK(qihse_vector_db_checkpoint(db)); + CHECK(qihse_vector_db_close(db)); + + db = qihse_vector_db_open(QIHSE_VECTOR_DB_INMEMORY, NULL, path, + QIHSE_VDB_OPEN_FILE_BACKED); + CHECK(db != NULL); + CHECK(qihse_vector_db_get_edges(db, 101u, "CALLS", neighbors, 2u) == 0); + CHECK(!qihse_vector_db_add_edge(db, 101u, 999u, "CALLS", NULL, 0u)); + CHECK(errno == ENOENT); + CHECK(qihse_vector_db_close(db)); + + CHECK(qihse_ctr_open_write(path, false, &ctr)); + CHECK(qihse_ctr_flush(&ctr, &remove_edges, 1u)); + qihse_ctr_close(&ctr); + db = qihse_vector_db_open(QIHSE_VECTOR_DB_INMEMORY, NULL, path, + QIHSE_VDB_OPEN_FILE_BACKED | QIHSE_VDB_OPEN_READ_ONLY); + CHECK(db != NULL); + CHECK(qihse_vector_db_get_edges(db, 101u, NULL, neighbors, 2u) == 0); + qihse_vector_db_destroy(db); + return 0; +} + +int main(void) { + const char* path = "/tmp/qihse_edge_persistence_test.qdb"; + pid_t child; + int status = 0; + unlink(path); + + child = fork(); + CHECK(child >= 0); + if (child == 0) return write_uncheckpointed_edges(path); + CHECK(waitpid(child, &status, 0) == child); + CHECK(WIFEXITED(status) && WEXITSTATUS(status) == 0); + CHECK(verify_replay_and_checkpoint(path) == 0); + CHECK(verify_remove_and_old_container(path) == 0); + unlink(path); + puts("qihse explicit edge persistence tests passed"); + return 0; +} diff --git a/tests/test_python_edges.py b/tests/test_python_edges.py new file mode 100644 index 0000000..6197ffd --- /dev/null +++ b/tests/test_python_edges.py @@ -0,0 +1,48 @@ +import os +import subprocess +import sys +import tempfile +import textwrap +import unittest + + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +class PythonEdgeBindingTest(unittest.TestCase): + def test_binding_in_fresh_subprocess(self): + with tempfile.TemporaryDirectory() as directory: + path = os.path.join(directory, "edges.qdb") + script = textwrap.dedent( + f""" + import sys + import numpy as np + sys.path.insert(0, {os.path.join(ROOT, 'python')!r}) + from qihse import VectorDB + + path = {path!r} + db = VectorDB.create(path, 2) + db.add_vectors(np.asarray([[1, 0], [0, 1]], dtype=np.float32), ids=[11, 12]) + assert db.add_edge(11, 12, "CALLS", b"line=7") + assert not db.add_edge(11, 12, "CALLS", b"ignored") + assert db.neighbors(11, "CALLS") == [12] + assert db.edge_records(11) == [(11, 12, "CALLS", b"line=7")] + db.replace_edge(11, 12, "CALLS", b"line=8") + db.close() + + db = VectorDB.open(path, read_only=True) + assert db.edge_records(11) == [(11, 12, "CALLS", b"line=8")] + db.close() + """ + ) + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = ROOT + completed = subprocess.run( + [sys.executable, "-c", script], cwd=ROOT, env=env, + text=True, capture_output=True, timeout=60, check=False, + ) + self.assertEqual(completed.returncode, 0, completed.stderr + completed.stdout) + + +if __name__ == "__main__": + unittest.main() From ae20f71909a47df117ffd6eab83bc7a42cf53be8 Mon Sep 17 00:00:00 2001 From: SWORDIntel Date: Tue, 14 Jul 2026 15:58:43 +0100 Subject: [PATCH 2/5] fix: keep embedded KV reads authoritative --- Makefile | 10 ++++++-- src/black_hole/qihse_kv_store.c | 31 ------------------------ tests/test_kv_read_integrity.c | 42 +++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 33 deletions(-) create mode 100644 tests/test_kv_read_integrity.c diff --git a/Makefile b/Makefile index ef4c1ed..d51118e 100644 --- a/Makefile +++ b/Makefile @@ -146,7 +146,7 @@ endif # because their functionality is already partially in qihse_math.c / qihse_search.c # or provided by qihse_exports.c stubs. -.PHONY: all build build-native clean pristine workspace workspace-clean lib lib-ctypes liboqs oqs-provider persistence persistence-check test benchmark install dev-setup docs test-persist test-edge-persistence test-trinary-codec test-memory-planner test-memory-topology-probe test-memory-planner-trace test-memory-allocation-policy test-memory-coherence test-memory-migration-policy test-memory-migration test-memory-device-placement test-memory-migration-backend test-memory-migration-scheduler bench-trinary-codec bench-trinary-db-candidate bench-micro bench-trinary-search-path bench-trinary-search-sweep bench-trinary-random-sweep bench-trinary-weighted-sweep bench-trinary-magnitude-sweep bench-reference-workloads bench-reference-runner-smoke sample-vxug-pdf-workload bench-vxug-pdf-workload bench-reference-workload bench-reference-result-summary bench-sift1m-workload bench-sift1m-fallback-data calibrate-sift1m-workload validate-reference-workflow check-upstream-workflow check-upstream-workflow-strict check upstream-pr-loop test-all-isa test-vnni-bench test-vnni-only test-avx2-only test-avx512-direct test-amx-only test-direct-execution test-simple-exec +.PHONY: all build build-native clean pristine workspace workspace-clean lib lib-ctypes liboqs oqs-provider persistence persistence-check test benchmark install dev-setup docs test-persist test-edge-persistence test-kv-read-integrity test-trinary-codec test-memory-planner test-memory-topology-probe test-memory-planner-trace test-memory-allocation-policy test-memory-coherence test-memory-migration-policy test-memory-migration test-memory-device-placement test-memory-migration-backend test-memory-migration-scheduler bench-trinary-codec bench-trinary-db-candidate bench-micro bench-trinary-search-path bench-trinary-search-sweep bench-trinary-random-sweep bench-trinary-weighted-sweep bench-trinary-magnitude-sweep bench-reference-workloads bench-reference-runner-smoke sample-vxug-pdf-workload bench-vxug-pdf-workload bench-reference-workload bench-reference-result-summary bench-sift1m-workload bench-sift1m-fallback-data calibrate-sift1m-workload validate-reference-workflow check-upstream-workflow check-upstream-workflow-strict check upstream-pr-loop test-all-isa test-vnni-bench test-vnni-only test-avx2-only test-avx512-direct test-amx-only test-direct-execution test-simple-exec .NOTPARALLEL: validate-reference-workflow all: liboqs oqs-provider lib server keygen @@ -231,7 +231,13 @@ test-edge-persistence: lib -L. -lqihse $(LDFLAGS) LD_LIBRARY_PATH=. ./tests/qihse_edge_persistence_test -test: test-omni test-e2e test-e2e-memory-planner test-persist test-bytecode test-document-store test-column-store test-fts-engine test-timeseries test-trinary-codec test-memory-planner test-memory-topology-probe test-memory-planner-trace test-memory-allocation-policy test-memory-coherence test-memory-migration-policy test-memory-migration test-memory-device-placement test-memory-migration-backend test-memory-migration-scheduler test-quantization +test: test-omni test-e2e test-e2e-memory-planner test-persist test-bytecode test-document-store test-column-store test-fts-engine test-timeseries test-trinary-codec test-memory-planner test-memory-topology-probe test-memory-planner-trace test-memory-allocation-policy test-memory-coherence test-memory-migration-policy test-memory-migration test-memory-device-placement test-memory-migration-backend test-memory-migration-scheduler test-quantization test-kv-read-integrity + +test-kv-read-integrity: lib + $(CC) $(CFLAGS) -o tests/test_kv_read_integrity tests/test_kv_read_integrity.c -L. -lqihse $(LDFLAGS) + rm -f qihse_integrity.chain* + @status=0; LD_LIBRARY_PATH=. ./tests/test_kv_read_integrity || status=$$?; \ + rm -f tests/test_kv_read_integrity; exit $$status test-bytecode: lib $(CC) $(CFLAGS) -o tests/test_bytecode tests/test_bytecode.c -L. -lqihse $(LDFLAGS) diff --git a/src/black_hole/qihse_kv_store.c b/src/black_hole/qihse_kv_store.c index 782d53e..ca4b448 100644 --- a/src/black_hole/qihse_kv_store.c +++ b/src/black_hole/qihse_kv_store.c @@ -97,18 +97,6 @@ static void recover_from_wal(qihse_kv_store_t* store) { fclose(f); } -static void qihse_kv_seed_honeypot_bait(qihse_kv_store_t* store) { - if (!store) return; - - for (int i = 0; i < 256; i++) { - char bait_key[64]; - char bait_val[128]; - snprintf(bait_key, sizeof(bait_key), "_qdd_bait_%04d", i); - snprintf(bait_val, sizeof(bait_val), "CLASSIFIED_DECOY_DATA_%d", i); - qihse_kv_set(store, bait_key, bait_val, 3, 0xFF); - } -} - qihse_kv_store_t* qihse_kv_store_create() { qihse_kv_store_t* store = (qihse_kv_store_t*)malloc(sizeof(qihse_kv_store_t)); if (!store) return NULL; @@ -143,9 +131,6 @@ qihse_kv_store_t* qihse_kv_store_create() { // Recover WAL recover_from_wal(store); - // Seed honeypot bait keys for quantum defense - qihse_kv_seed_honeypot_bait(store); - // Rotate WAL — start fresh, archive old log for debugging rename(WAL_PATH, QIHSE_DATA_DIR "wal.log.old"); #ifndef _WIN32 @@ -267,14 +252,12 @@ char* qihse_kv_get_user(qihse_kv_store_t* store, const char* key, qihse_user_t* qihse_kv_sweep_expired(store); size_t out_size = 0; void* val = qihse_trinary_trie_search(store->trie, key, &out_size); - bool user_authorized = true; if (val) { // Find auth info in memory for (size_t i = 0; i < store->num_keys; i++) { if (strcmp(store->keys[i].key, key) == 0) { if (!qihse_auth_can_access(user, store->keys[i].classification, store->keys[i].sci_compartment)) { - user_authorized = false; val = NULL; // Masked: pretend it doesn't exist in MemTable } break; @@ -282,20 +265,6 @@ char* qihse_kv_get_user(qihse_kv_store_t* store, const char* key, qihse_user_t* } } - if (user_authorized && val) { - qihse_qdd_response_tier_t tier = qihse_qdd_get_response_tier(store->qdd_ctx); - - if (tier == QIHSE_QDD_RESPONSE_THROTTLE) { - usleep(100000); - } else if (tier == QIHSE_QDD_RESPONSE_HONEYPOT) { - static char honeypot_buf[128]; - qihse_qdd_generate_honeypot_response(honeypot_buf, sizeof(honeypot_buf)); - return strdup(honeypot_buf); - } else if (tier == QIHSE_QDD_RESPONSE_ACTIVE) { - return NULL; - } - } - if (val) return strdup((char*)val); // LSM-Tree: Search SSTables (from newest to oldest) diff --git a/tests/test_kv_read_integrity.c b/tests/test_kv_read_integrity.c new file mode 100644 index 0000000..029c490 --- /dev/null +++ b/tests/test_kv_read_integrity.c @@ -0,0 +1,42 @@ +#include +#include +#include + +#include "qihse_auth.h" +#include "qihse_kv_store.h" + +#define READ_ITERATIONS 512 + +int main(void) { + qihse_kv_store_t* store = qihse_kv_store_create(); + if (store == NULL) { + fprintf(stderr, "could not create KV store\n"); + return 1; + } + qihse_auth_init(); + qihse_user_t* user = qihse_auth_get_user(0); + if (user == NULL) { + fprintf(stderr, "could not obtain test user\n"); + qihse_kv_store_destroy(store); + return 1; + } + if (!qihse_kv_set(store, "memory_record", "authoritative_value", 0, 0)) { + fprintf(stderr, "could not write KV record\n"); + qihse_kv_store_destroy(store); + return 1; + } + + for (int iteration = 0; iteration < READ_ITERATIONS; ++iteration) { + char* value = qihse_kv_get_user(store, "memory_record", user); + if (value == NULL || strcmp(value, "authoritative_value") != 0) { + fprintf(stderr, "read %d returned non-authoritative data\n", iteration); + free(value); + qihse_kv_store_destroy(store); + return 1; + } + free(value); + } + + qihse_kv_store_destroy(store); + return 0; +} From 817831d2184a285543de6382c2f9bbf0be3d5dc0 Mon Sep 17 00:00:00 2001 From: SWORDIntel Date: Tue, 14 Jul 2026 16:47:18 +0100 Subject: [PATCH 3/5] feat: add unified Linux launcher Document the launcher, enable strict-C99 PQC declarations, and stop tracking generated Python and WAL artifacts. --- .gitignore | 4 + README.md | 15 + python/qihse/__pycache__/core.cpython-313.pyc | Bin 15656 -> 0 bytes qihse | 29 ++ qihse_launcher.py | 345 ++++++++++++++++++ wal.log.old | 1 - 6 files changed, 393 insertions(+), 1 deletion(-) delete mode 100644 python/qihse/__pycache__/core.cpython-313.pyc create mode 100755 qihse create mode 100644 qihse_launcher.py delete mode 100644 wal.log.old diff --git a/.gitignore b/.gitignore index e719a44..673f1cf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Root-level build artifacts and binaries /*.so /qihse-server +/qihse_keygen /test_mldsa /fuzzer @@ -11,10 +12,13 @@ # Root-level cache, logs, and temp files /qihse_qmag_telemetry.log /wal.log +/wal.log.old /qihse_audit.log /qihse_integrity.chain /.DS_Store /__pycache__/ +**/__pycache__/ +*.py[cod] # CNSA 2.0 PQC key material (generate with scripts/qihse_keygen.sh) /qihse_kem_key.pem diff --git a/README.md b/README.md index 062b192..4b26e52 100644 --- a/README.md +++ b/README.md @@ -172,6 +172,21 @@ make test make bench-micro ``` +The repository also includes a unified Linux launcher for the common build, +status, test, benchmark, database, and SDK workflows: + +```bash +./qihse status +./qihse build +./qihse test +./qihse db --help +./qihse python +``` + +Run `./qihse --help` for the complete command list. The launcher delegates to +the existing Make targets and scripts; it does not install a daemon or modify +system configuration. + ### Minimal C Example ```c diff --git a/python/qihse/__pycache__/core.cpython-313.pyc b/python/qihse/__pycache__/core.cpython-313.pyc deleted file mode 100644 index 8b639a43021fa590e70315932108e050dedffa5f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15656 zcmdrzYj9IndiP3K($&L~EZg$imLJ#%Yzu>V1Omihn@7M93lV~yD6%9Q1zB>AWD>mD z1a>=#*^os}HjBNpQ_OZ}fN5p}+q96Gwi`M-n`EXlD@AmD_r^`>G&3FknV6KB{nLKm zxw^WtBC~0`(;qz^o#%HR_k7Ruo#UmVA_D_qdtdVlM_U=@->@JDVmV}E9F;8-I|Rm31Wqn7XJDcc?KnIWHx3B z%9w7AOHE4MDpEF7TEgtz%rL~h!aOczBsn)^mshv?OWtpC55@F<f&JL14>!W<*ls{gtE3;7Yn2G%YBtp8O5zy0U~D*&dG3A_$H|RAW z*D2|WwJXhVMCmK%a%Nbm*Ap_Vl75R`1=cI~a_L~7jw++cIb|o|%Z|ylO1eU??Cw(9 zHUe#%p#P&drJX_+0i^klKz0Dtu7t66`FmD`=*WR6upg8zpHse1&Mn)ALVQ^^P^edS zALyD3%m?c%P)j-Yd3kOnUABtC*eKL1XBCuVhLsWJ+_xCHP_O)v1M+xE`YlE-)GKFX z*rO3;JUIuoE6KbQGK`Xbi&Yfrm0bm;P76?{Rn~Z9BdsfjI$O4G6Rj(Oy3%aj=24Zi zY;uR;seOChy^h1D<0qmKhd+LLJQ#DlM114pLE;#R5=S^R{9@=tEZ7l?I!*=sai|LT z;=W-YR2lB!r}R5nQMWG=?~P21IXRI#5Q@b`{?K?l6pi@8qUOlNcsM9(JP{~Q8gs^* zT$@vHM51w+a3T`uaO|20g#(V4Lh%z@9b>+eL0p7i*)@jA7y!&tDb&8p3^8CVF=!}( zk)$Hrs2bLx_a3Fo?b&q?ZJ2AYPL0TWy%FD7(CZZqUhh~mFcHSQ$?JV_!WYi8SiIhm z5Q)XZp-3I>9w(vUiFh#P^%At)mT*;~%QfPQ#a#z|p@`$@C^;D$ z_xXb^y0flvy0b32hc17V1Utr06Fu~z5w0wzf#`KcIMn0WeMHpvdUhY!*VEhGPpk-uUoWlKhB-t8jc~FUe#h&KP-H&SC)i5P zJSkXors15q9qjZhbIETOwX8Tl`F=AaUTj4(LNCidVyXeKA6Doiv&VY9gK;V zY~Je!@emgc(i(t!gJOx&I3A6Ly|K__P_zUx^zVhifdNE|JP9y%I36UTJ+HzW9rxj= zNYwq#qQYDu2}e|ldc43ZN zk+WS|VsP`sII4FV#G(w33|~SIo3ra?Jwm ztywJ6DKvnSEu#S*DljbdLzG7Ydc-wgRpZtSX@_)$FodeHOY7EFF)wg0utbl?+sy8( zVE0XufXZzCO90$YHbG3_RL&4QruaFC%s_0 z!zgo&?Z2kZ8x_r*STH=2R~^I291sYS6dD_$NGOgLMD98}e!OXZ-(2`xg((YDU7&o^;)66(E{gc+* z1+5MRwomeH4ob4yC)FKYBO>c`s+6z0#*$qyR18<7d>jdmlL8+nReu-Z8zF*^D{h!; zzS4TRb*}R2>g!!Mx8B%#%b#4o>!XIGr6nGK5!By$rqxqjjn`PJlylc`Pn zzsBmX4Xmb@m6nISmr*tvO&FGti1H{~1yZGe-QrfmAJ;|cy^sX}b_S0Y1vqf@Zaw@A zLxwJs8xE?J`i*Ybnb9JrSu{%MlN9q=nExKm*7xra(LhO`I3O=-LDYZ_B!~*qgi!-T zepxm{UZJf76y9Y9v5tm9bkxXr*c^yl_R92fvyTA{hLKun0M^Pv0c#=dr1~eq+HC%C zIJ=@_&xd8H!HhCaRsS_s)CBNk;?$D^&_mSuVifD5j*`QM^~(1WHvr^)tW!s+_t8Kx zoF91?0%c!HQetV&PG=eSdjJ|kIdg`2U}MTFFTU{h3v>4O$orGmCR6sUQ`~u@%*&9g z@3_O)2#$4k_*$WP%RliAl%Ski2^$nIbkT?xiZi;<70eLH(5^sG+zN6KBlKj699kaa zNi0)lD5MY$VJk*RGcg}TuQQ80W3LbzT=NZ^QVp8~$10(rMQCx}x0p4L-L={-8m0}4 z8ptnd8EeU+4s$$HT)(KtoPnvTTQp+M#8lTW7GciJ)HgrG-ir)G68GSo%ecn`6&RLY zWeaf++5t%Q1T+I^2pwn$-mP_`E&vVi0w=p=i5}V;Cy2<8A~y*fQXt>sS`KEkngawjmx#6@P9kEth1=T6a+w?}kG*<;zkW4aRG7 z+lI;tk2qxSQqhr0U?e)OyA*Qew5|+S4)axDsmtKDXU45`H@nN>yDR9o7s0nzW?HJ; ztK3zXC)JRvrn#D-T9hBKb2V7betn(0#a#;{)WI%lL7r&c^_fxX-45aadD1HK5|zVI z`5jn-s2-0KzY214z&!Hv66&1giW4=FqB;k&oIa4{ZFvv|+|%-^pidT(@;E)8!U3K0 z!rY=pI#THI0O1RSBBP>a;6T)W5*osgfNI(WqZ0IO#c*xQShiUHm@L2k2;tB?1zl8H zeqJw>*?~t53uSc|y>ENdWi36OU@sw2LWW=&1OL zcSsep03Ji-eTMncY@0VbQ)cJYhO0-wR@io9+pU+<8}_8ld#BW2SZoWnx@5zSTk%^X z$@<>3tuJZnTPP|^_}_eXN+pz(pXUWj@swV|r5=gL8;yid7e?WGFe92$u=T9G^*%tB zN99(vmoiL5m80nXd+1fm=mi922tA_qsO4Gm5X?OS!P$|h;YJZk?67e10z{M53QnWr z!N_uqYRE}|5lizCijZUs$B1BrD3q~Bumc!bzR{S(UAXwHN)$QlDi&qYhsB=(L<|+! z8Ro9FcuMnyu~aY>Cyu8~^;5kIh|boOx%Dc0wd;E6&FUM~x1LUS_I@;xHXl?F`uMFQ zxB8OxyVJIwq^W1YQa+=8^SepDhT?IV9gI?i^55S>=3V42Nb`e=abOEdhP)snz}EsC zmO%`|3}kst0WaKNMk`#n!VHSsb3@t9$j(&<*ewlA8zava03A1?axnFfToismdfXc>t*wgDoLbw$M&j=yssL(rs4X zHXFKYbjx-gzCqe~L;>!6vt+JIp1}OY>Al`y1Y9j}KCopMn63LI3_%SpSfr(7s=#>% zafeu0YR=3yfb-`CS)1UVO1sIDNr1`d2a)rU(O9m6z$01kWqKsb)e=(GUqes21F$IG zbir`W@Ro@ZV85t4MP11;ID`aZ6$sqt zynv>4g!Qlkd|m?--Kwo{_u{jG{9*?Xk$J%lCY4N}bPU-9%YmDK*~fd2j(h_dw!8E4d^OKzMs$@tdZOr{LG!pUxe3LToRuL5cp|~fCu0#1m5E@QHS1V-#FYo(S({Q)SpJSAtd#kp%+cm2jS`} zNWdf#IX~Rwh^3NU0@ps^j~?|7PmGL!*H|3}y-8(sb-s*-CCRRBD#}G2`n0{inB=R& zt)a>md>gwk|9y=93?mdKA{(K0lc)pn;)SEFs7D7On!+*BK(pxBj)`nYWXD8R^kfW~ z7xl6nB#)CZXC3mHnR;C#Dcwoo?y#AP~9}#Fr@6;ruGOH z`-NA|y)yG++TswZ8qn7V7TG|$YF$Dn*lRD2O^==1Gu16r)}G&g*ItDkZ0BBEsBXBl zb9U#|r#`FhxL%U1+L+MYt*DvW2VhhC1xtC#(zH<9bjdsGz53#3wd>L~8xs13y5>vK z+30olv%1c7ZCApuP_t_8#P4cc(>;mqg(l~fZ9m)g-l=)l{?A@Zs!NAA31mB10ZKp&W1F_{N8`UwUk>;`DNJ&{8J%p{49T zc;Gr0H7Y=qXCbsuLe~!s0HMo34k2ZTwGI-#6Wh?Bf54Fl3xu;t%LkC$G zXrIA-a*Lv0LF?}Y&1CWz2cDE$m2?m+0x1llKZ=H9KSd^e)QhS?JloKafjuIZl~i7t z%%yVK!5jqN9O`U%ti(v>LW-H_fRt2L$abr!pS0c&`10y z93%9M?_5jOqG$-kLJ{dL0CoEnbb&6{ojDH1w{Uc%6-jAcz+4+fRGrmf4s9IjP?nsT z%~)K85uC3Wf>w#BIpqt()hY@u;=~B84SFjj>C$ib8)EcJBsHoTK23rnRCS6P;){&J zl^VWHi3F%SpL#x}YYN0Sz4n3YTdHDbvEnBfAp)sOSma&=L!LOWJPHwB;YwoL7@dfF z!CZhlJh;i0?wQeiz%uy}fX0d-l9eq@C5@|@z3nS+lHXEtNB?eqKFq zX-`?&h4LD_bWLzVMeQZytTCYxD(f!Q%+@5dLSyTE_ZaO-(=Z?uCG`8F^)eAKZ=MM^NJFsfC(6aH4 zsYz&U!&0-*z8=#xLd)8#zRMxZHwg`mnAQnvT-O_~t-*Y&(9(7#d^s$&x6ryKp}iB! zYrd>reRbn!)$7t#8>XK8vb18p)RiiAWpwm{)qc_RwrS?+v~`tWtro0hOL|S2d8!v| z6l+=H#WzobE}q+&wr@yUH{7*XOzl}HtD4#P$+3xy3=jD ze%<@Y+D|*?_dl1~|6IDqo34NUcUIqhHDhnQTT=DV!%|It0nWTslkaKY%c(!%40~-l zkv%9{f+O(Q062$C`0*K&KdX}gt*CRUCjT4AJW`W`){{aZ7a+*Y0^0kSE$a}>^%tQe z%sTzttSn!seGT3!a3^Nwf}^nN5mH4ZxgMPA$vWwgRs2NIF$&MUPB=o5K=5TpEEEX( zeWWATvouVfW92}>ODf6iJ_(;Ee#021@Lmc}xmdD%If*U*61tH0F#0h>c@9EjPS+g! zHK(>1SSp5hao}x|+#3tUWs<#jz4@l|hV#}_>EgpvyS^wXqvqQaX?u6l+AW!HnymQ- z`^mhuq1&LoZ7_6KEO&Hd@%sS){i*RQV-8Q?vMonXPk&nMuH8j(hS-J2DqG-c*9UKM z9Pr8we7BSEmJJ^JWl>$F9Ifb1OI;{EEXQ)@jCDvR^F&g7jIJ#3&P-ur1gnB#4b_9uJ6{XZ|e=+<^TT> z+k7}kV(`4l;R}Zys0ZPE>hPaH%@{+^K0JLo*)eJA3s1ykV_-QB%kptJf}jfe$X@{l zmif%U9SjgTenwiZM{I5V2Da9wiq~H4d4J!ved*#&GPdS+r0wgI*7bSFA}Bx$`xwzr zvl8oN{EkjmI?$tmnBx|e6uaW0#+#fLTr0|Xpe%5PPM(`g@|E&^Pn5(((=brG7mmrG z@?*<{JrBL8uooV&BJ?-sxb||7eb5U-VPQXrqtj66V1~ylSzLHj189Kc1w|3aN&}3{ zJazC$BU4ca8{Kgqi316N#5lYM-rka+W1K|eQGYb-hy}?hcn%D&Td{fA=RY}0=u2IC z3DFPt>j7AqWX@$|1Q?wp?j0ZX;z(X;0Ex)f0;v3;sdE#(hj6MT-SBrf3Oe>CFS!a>r*A`bF2)@+m<L#2+P0+Q zO3e?|%WZ;guDe=&yV}t0)WHIMxHkHk&qL11_QAu1ey6&>A2RpQC3o)=tXGi}kn}rM zp)GK-o78u%-`ued{yLovg`Rhb#_TI)?>N~FT}2ao6}&YDug{5SKvaVbB3ZfP&`6GB zbPA(MjJ}J}S&R-~bPyw?K9LLKBW$pq;RPkRh}Alb@Yo|6_^3k3HH_ZJ2vr^V5F(KW zti#*y_-W}T?;o%MKTmS=7h?{Oywb(r5aiy0Uu+UKAD$AxlbY6IE&y~sc^r>4nO!xEO3)N=Ep6>|4gI!$c?9^*hV*F}@-`yXH_0|>dVY@DXyD-)|J z{YuMf%D^;iUM;NYxUX;4Jbt&hw>or_l=#J>bsWG#K6Vl)5qtklVxpSt1oJx7KSyI zCU&3OzNo_zyn(2ksY_W`FY2)WTVrO6vFuetvfaynp zuW9HwG2hxvljOkBhghTqD$1mL zye*>6>kUNx@IIX~nkovH7O~TCr~wxzyga9LG7KmaIdpmwWHeC=k841W5Dp5IehAt? z63L8E*cYOq2>3KZ1mDQR`X0dMK#N^_f;yETjCkk{5hlloH2aP z*gj{h|IAeU3*$;NE}$Qc?u|WX_q=i7?16;sV)=CW#ro;`Iop-;%jH+rMWhN zD_+!bX4M%3kR)$9!+owVInyoZ^l!X$_N6yoJ^O0HH?ONm>na3G*@f@?;5##&Z@xOk z{hlwnuVW0QK+qh2_VM2v%98foX+uwv>v>>eS$#q~Z*`=sjt2}B;3J@KH5gc=#A({r#^n|_G`)3r_XTnOm&K>zQZ&NEuBiK z>8^Bmyxw>7(2YYM9sSsQ+nYQ(G=DUhIvPx_8bQd)6jOPJX%w1WN~z(lv+HURmTOW> z%^hZy(7IMBH3QdbtO93vqyKyT_`Y>|N1AKI@}aYbauTU^=k(4r*Q7ufE-za6b`?(H zNHLB(j1#`JZTNTtBT$wEcRJOL`85`%4>YDDhZ@Ja-E7ymvPd5E_=chZ9ZG*}} zoWk19yrtc3+kBNSwLZnv-(f(6u6w`zT08cUYFh8MuAdvja#f0{y2CUHtJYq5_VTkz zO(TqP%l@nS59>c^`K{|yS90xfrF($^A96ajm%Yzm{IJx*K6W2= int: + """Run a command, streaming output to the terminal. Returns exit code.""" + full_env = os.environ.copy() + if env: + full_env.update(env) + full_env.setdefault("LD_LIBRARY_PATH", str(ROOT)) + return subprocess.call(cmd, cwd=cwd or str(ROOT), env=full_env) + + +def _run_make(target: str, *extra: str) -> int: + return _run(["make", "-j", str(os.cpu_count() or 4), target, *extra]) + + +def _lib_info() -> dict: + info: dict = {"path": str(LIBQIHSE), "exists": LIBQIHSE.exists()} + if not LIBQIHSE.exists(): + return info + info["size_kb"] = LIBQIHSE.stat().st_size // 1024 + try: + import ctypes + lib = ctypes.CDLL(str(LIBQIHSE)) + if hasattr(lib, "qihse_version"): + lib.qihse_version.restype = ctypes.c_char_p + info["version"] = lib.qihse_version().decode("utf-8", "replace") + if hasattr(lib, "qihse_build_info"): + lib.qihse_build_info.restype = ctypes.c_char_p + info["build_info"] = lib.qihse_build_info().decode("utf-8", "replace") + if hasattr(lib, "qihse_available"): + lib.qihse_available.restype = ctypes.c_bool + info["available"] = lib.qihse_available() + except Exception as exc: + info["load_error"] = str(exc) + return info + + +# ── Command handlers ───────────────────────────────────────────────────── + +def cmd_build(args: list[str]) -> int: + """Build libqihse.so via make (auto-detects SIMD features).""" + print("[qihse] Building libqihse.so ...") + return _run_make("lib") + + +def cmd_build_ctypes(args: list[str]) -> int: + """Build ctypes-only variant (no Python C extension linking).""" + print("[qihse] Building libqihse.so (ctypes-only) ...") + return _run_make("lib-ctypes") + + +def cmd_build_native(args: list[str]) -> int: + """Build via build-native.sh with full SIMD auto-detection.""" + print("[qihse] Building via build-native.sh ...") + script = SCRIPTS / "build-native.sh" + if not script.exists(): + print(f"[qihse] ERROR: {script} not found", file=sys.stderr) + return 1 + os.chmod(script, 0o755) + return _run([str(script)] + args) + + +def cmd_build_tui(args: list[str]) -> int: + """Launch interactive build configuration TUI.""" + print("[qihse] Launching build TUI ...") + return _run([sys.executable, str(SCRIPTS / "build-tui.py")] + args) + + +def cmd_test(args: list[str]) -> int: + """Run the full test suite.""" + print("[qihse] Running test suite ...") + return _run_make("test") + + +def cmd_bench(args: list[str]) -> int: + """Run the benchmark suite (validate-reference-workflow).""" + if args: + return _run_make(*args) + print("[qihse] Running benchmark suite ...") + return _run_make("benchmark") + + +def cmd_db(args: list[str]) -> int: + """QIHSE vector DB CLI — delegates to scripts/qihse-db.""" + script = SCRIPTS / "qihse-db" + if not script.exists(): + print(f"[qihse] ERROR: {script} not found", file=sys.stderr) + return 1 + return _run([sys.executable, str(script)] + args) + + +def cmd_server(args: list[str]) -> int: + """Build and run the QIHSE test server.""" + print("[qihse] Building and running test server ...") + return _run_make("server") + + +def cmd_keygen(args: list[str]) -> int: + """Build and run the PQC key generator.""" + print("[qihse] Building qihse_keygen ...") + rc = _run_make("keygen") + if rc != 0: + return rc + keygen = ROOT / "qihse_keygen" + if keygen.exists(): + return _run([str(keygen)] + args) + print("[qihse] qihse_keygen binary not found after build", file=sys.stderr) + return 1 + + +def cmd_demo(args: list[str]) -> int: + """Run the Python SDK demo.""" + print("[qihse] Running Python SDK demo ...") + env = {"LD_LIBRARY_PATH": str(ROOT)} + full_env = os.environ.copy() + full_env.update(env) + full_env["PYTHONPATH"] = str(ROOT / "python") + os.pathsep + full_env.get("PYTHONPATH", "") + return subprocess.call( + [sys.executable, str(SCRIPTS / "qihse_python_demo.py")] + args, + cwd=str(ROOT), env=full_env, + ) + + +def cmd_python(args: list[str]) -> int: + """Start a Python REPL with qihse importable.""" + env = os.environ.copy() + env["LD_LIBRARY_PATH"] = str(ROOT) + os.pathsep + env.get("LD_LIBRARY_PATH", "") + env["PYTHONPATH"] = str(ROOT / "python") + os.pathsep + env.get("PYTHONPATH", "") + cmd = [sys.executable] + if args: + cmd.extend(args) + else: + cmd.append("-i") + cmd.append("-c") + cmd.append( + "import qihse; print(f'QIHSE {getattr(qihse,\"__version__\",\"?\")} loaded'); " + "print('Available: VectorDB, QihseDB, DistanceMetric'); " + "print('Try: db = qihse.QihseDB(); db.kv_set(\"k\",\"v\"); db.kv_get(\"k\")')" + ) + return subprocess.call(cmd, cwd=str(ROOT), env=env) + + +def cmd_bootstrap(args: list[str]) -> int: + """Initialize workspace directories.""" + print("[qihse] Bootstrapping workspace ...") + script = SCRIPTS / "bootstrap-workspace.sh" + if not script.exists(): + print(f"[qihse] ERROR: {script} not found", file=sys.stderr) + return 1 + os.chmod(script, 0o755) + return _run(["sh", str(script)] + args) + + +def cmd_clean(args: list[str]) -> int: + """Remove build artifacts.""" + print("[qihse] Cleaning build artifacts ...") + return _run_make("clean") + + +def cmd_pristine(args: list[str]) -> int: + """Deep clean — build artifacts, data, and results.""" + print("[qihse] Pristine clean (build + data + results) ...") + rc = _run_make("clean") + if rc != 0: + return rc + for d in ("data", "results", "build"): + p = ROOT / d + if p.exists(): + print(f" rm -rf {p}") + shutil.rmtree(p) + print("[qihse] Pristine clean done.") + return 0 + + +def cmd_isa_info(args: list[str]) -> int: + """Show CPU ISA detection and build flags.""" + return _run_make("isa-info") + + +def cmd_check(args: list[str]) -> int: + """Run upstream workflow checks.""" + print("[qihse] Running workflow checks ...") + return _run_make("check") + + +def cmd_dev_setup(args: list[str]) -> int: + """Check required toolchain.""" + print("[qihse] Checking toolchain ...") + return _run_make("dev-setup") + + +def cmd_version(args: list[str]) -> int: + """Show library version and build info.""" + info = _lib_info() + if not info.get("exists"): + print("QIHSE library not built. Run: ./qihse build") + return 1 + print(f"QIHSE {info.get('version', 'unknown')}") + print(f" path: {info['path']}") + print(f" size: {info.get('size_kb', '?')} KB") + print(f" build_info: {info.get('build_info', 'n/a')}") + print(f" available: {info.get('available', '?')}") + if "load_error" in info: + print(f" load_error: {info['load_error']}") + return 0 + + +def cmd_status(args: list[str]) -> int: + """Show build status, library info, and availability.""" + info = _lib_info() + print("╔══════════════════════════════════════════════════╗") + print("║ QIHSE Vector Engine — Status ║") + print("╚══════════════════════════════════════════════════╝") + print() + print(f" Root: {ROOT}") + print(f" Library: {info['path']}") + print(f" Built: {'✓' if info['exists'] else '✗ (not built)'}") + if info.get("exists"): + print(f" Version: {info.get('version', '?')}") + print(f" Size: {info.get('size_kb', '?')} KB") + print(f" Available: {info.get('available', '?')}") + if "load_error" in info: + print(f" Load Error: {info['load_error']}") + print() + + # Check toolchain + tools = {} + for tool in ("gcc", "make", "python3", "git"): + tools[tool] = shutil.which(tool) is not None + print(" Toolchain:") + for tool, ok in tools.items(): + print(f" {tool:12s} {'✓' if ok else '✗'}") + print() + + # Check Python bindings + py_bindings = ROOT / "python" / "qihse" + print(f" Python SDK: {'✓' if py_bindings.is_dir() else '✗'}") + + # Check scripts + scripts = list(SCRIPTS.glob("*.py")) + list(SCRIPTS.glob("*.sh")) + print(f" Scripts: {len(scripts)} files in scripts/") + print() + if not info.get("exists"): + print(" → Run './qihse build' to build the native library.") + return 0 + + +# ── Dispatch ───────────────────────────────────────────────────────────── + +COMMANDS = { + "build": cmd_build, + "build-ctypes": cmd_build_ctypes, + "build-native": cmd_build_native, + "build-tui": cmd_build_tui, + "test": cmd_test, + "bench": cmd_bench, + "db": cmd_db, + "server": cmd_server, + "keygen": cmd_keygen, + "demo": cmd_demo, + "python": cmd_python, + "bootstrap": cmd_bootstrap, + "clean": cmd_clean, + "pristine": cmd_pristine, + "isa-info": cmd_isa_info, + "check": cmd_check, + "dev-setup": cmd_dev_setup, + "version": cmd_version, + "status": cmd_status, +} + + +def _help() -> None: + print( + "QIHSE Launcher — Unified entry point for the QIHSE vector engine\n" + "\n" + "Usage: ./qihse [command] [args...]\n" + "\n" + "Commands:\n" + " build Build libqihse.so (auto-detect SIMD)\n" + " build-ctypes Build ctypes-only variant (no Python extension)\n" + " build-native Build via build-native.sh (full SIMD auto-detect)\n" + " build-tui Interactive build configuration TUI\n" + " test Run full test suite\n" + " bench Run benchmark suite (or pass a specific bench-* target)\n" + " db QIHSE vector DB CLI (e.g. ./qihse db create --dims 128 --path /tmp/db)\n" + " server Build and run the test server\n" + " keygen Build and run PQC key generator\n" + " demo Run Python SDK demo\n" + " python Start Python REPL with qihse importable\n" + " bootstrap Initialize workspace directories\n" + " clean Remove build artifacts\n" + " pristine Deep clean (build + data + results)\n" + " isa-info Show CPU ISA detection and build flags\n" + " check Run upstream workflow checks\n" + " dev-setup Check required toolchain\n" + " version Show library version and build info\n" + " status Show build status and availability (default)\n" + ) + + +def main() -> int: + args = sys.argv[1:] + if not args: + args = ["status"] + + cmd = args[0] + rest = args[1:] + + if cmd in ("-h", "--help", "help"): + _help() + return 0 + + handler = COMMANDS.get(cmd) + if handler is None: + print(f"Unknown command: {cmd}", file=sys.stderr) + _help() + return 1 + + return handler(rest) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/wal.log.old b/wal.log.old deleted file mode 100644 index a9b0c3c..0000000 --- a/wal.log.old +++ /dev/null @@ -1 +0,0 @@ -SET test_key test_value 0 0 From 3bc2cabb875eaee69d8c178690fb62eb3197d723 Mon Sep 17 00:00:00 2001 From: SWORDIntel Date: Tue, 14 Jul 2026 16:47:49 +0100 Subject: [PATCH 4/5] chore: ignore local agent state --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 673f1cf..ea6aed7 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,12 @@ **/__pycache__/ *.py[cod] +# Local agent indexes, memory, and personal integration state +/.devin/ +/.learn/ +/.qlearn/ +/memory-bank/ + # CNSA 2.0 PQC key material (generate with scripts/qihse_keygen.sh) /qihse_kem_key.pem /qihse_kem_pub.pem From a92c828f216ccf831d5b66a4b0d73b115f86d0a4 Mon Sep 17 00:00:00 2001 From: SWORDIntel Date: Tue, 14 Jul 2026 17:06:19 +0100 Subject: [PATCH 5/5] fix: detect host ISA before enabling SIMD Select optional vector instruction sets from the host CPU flags so compiler support cannot produce illegal instructions at runtime. Keep machine-readable callers clean by routing the hardware profile to stderr and removing temporary vector DB diagnostics. --- Makefile | 19 +++++++++---------- src/broad_oak/qihse_system_guard.c | 7 +++++-- src/broad_oak/qihse_vector_db.c | 7 ------- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index d51118e..d2da484 100644 --- a/Makefile +++ b/Makefile @@ -10,25 +10,24 @@ QIHSE_CFLAGS_EXTRA?= # --------------------------------------------------------------------------- # CPU ISA feature flags # --------------------------------------------------------------------------- -# Each flag defaults to auto-detect via compiler probe at make time. +# Each flag defaults to auto-detect from the build host's advertised CPU flags. # Override on the command line or environment, e.g.: # make QIHSE_ENABLE_AVX2=1 QIHSE_ENABLE_AVX512=0 QIHSE_ENABLE_AMX=0 # -# Hosts without a feature MUST set it to 0; the build will not crash but the -# corresponding sources/flags are simply omitted. +# Cross-builds may override each flag explicitly on the make command line. # # R320/E5-2450 v2: AVX only – AVX2, FMA, AVX-512, VNNI, AMX all absent. # Sapphire Rapids+: all features available. -# ---- probe helpers --------------------------------------------------------- -# Returns "1" if the compiler can assemble the given flag, "0" otherwise. -cc_supports = $(shell echo 'int x;' | $(CC) $(1) -x c - -c -o /dev/null 2>/dev/null && echo 1 || echo 0) +# ---- host feature helpers -------------------------------------------------- +HOST_CPU_FLAGS ?= $(shell awk -F: '/^flags/{sub(/^ /, "", $$2); print $$2; exit}' /proc/cpuinfo 2>/dev/null) +cpu_has = $(if $(filter $(1),$(HOST_CPU_FLAGS)),1,0) # ---- per-ISA defaults (auto-detect unless already set in env/CLI) ---------- -QIHSE_ENABLE_AVX2 ?= $(call cc_supports,-mavx2) -QIHSE_ENABLE_AVX512 ?= $(call cc_supports,-mavx512f) -QIHSE_ENABLE_AVX_VNNI ?= $(call cc_supports,-mavxvnni) -QIHSE_ENABLE_AMX ?= $(call cc_supports,-mamx-tile) +QIHSE_ENABLE_AVX2 ?= $(call cpu_has,avx2) +QIHSE_ENABLE_AVX512 ?= $(if $(and $(filter avx512f,$(HOST_CPU_FLAGS)),$(filter avx512dq,$(HOST_CPU_FLAGS)),$(filter avx512bw,$(HOST_CPU_FLAGS)),$(filter avx512vl,$(HOST_CPU_FLAGS))),1,0) +QIHSE_ENABLE_AVX_VNNI ?= $(call cpu_has,avx_vnni) +QIHSE_ENABLE_AMX ?= $(if $(and $(filter amx_tile,$(HOST_CPU_FLAGS)),$(filter amx_int8,$(HOST_CPU_FLAGS)),$(filter amx_bf16,$(HOST_CPU_FLAGS))),1,0) # --------------------------------------------------------------------------- # Security & Audit Configuration diff --git a/src/broad_oak/qihse_system_guard.c b/src/broad_oak/qihse_system_guard.c index 018622d..92d0331 100644 --- a/src/broad_oak/qihse_system_guard.c +++ b/src/broad_oak/qihse_system_guard.c @@ -43,8 +43,11 @@ void qihse_system_guard_profile(void) { g_guard_initialized = true; - printf("[QIHSE System Guard] Profiled hardware on startup: %d cores, %zu MB physical RAM. DDR Bandwidth Est: %zu MB/s\n", - g_cpu_cores, g_system_ram_bytes / (1024 * 1024), g_memory_bandwidth_estimate_bps / (1024 * 1024)); + fprintf(stderr, + "[QIHSE System Guard] Profiled hardware on startup: %d cores, " + "%zu MB physical RAM. DDR Bandwidth Est: %zu MB/s\n", + g_cpu_cores, g_system_ram_bytes / (1024 * 1024), + g_memory_bandwidth_estimate_bps / (1024 * 1024)); } bool qihse_system_guard_check_operation(size_t required_bytes, bool is_brute_force) { diff --git a/src/broad_oak/qihse_vector_db.c b/src/broad_oak/qihse_vector_db.c index dd45ba4..abac7a4 100644 --- a/src/broad_oak/qihse_vector_db.c +++ b/src/broad_oak/qihse_vector_db.c @@ -4524,13 +4524,11 @@ qihse_vector_db_t qihse_vector_db_open( bool loaded = false; if (use_mmap && !read_only) { - printf("DEBUG: qihse_vector_db_open failed at %d\n", __LINE__); errno = EINVAL; return NULL; } vdb = (qihse_vector_db_t)calloc(1u, sizeof(*vdb)); if (!vdb) { - printf("DEBUG: qihse_vector_db_open failed at %d\n", __LINE__); errno = ENOMEM; return NULL; } @@ -4593,7 +4591,6 @@ qihse_vector_db_t qihse_vector_db_open( } if (has_manifest) { if (!qihse_vdb_load_snapshot(vdb, use_mmap)) { - printf("DEBUG: qihse_vector_db_open failed at %d\n", __LINE__); qihse_vector_db_destroy(vdb); return NULL; } @@ -4603,26 +4600,22 @@ qihse_vector_db_t qihse_vector_db_open( return NULL; } } else if (!create && !has_wal) { - printf("DEBUG: qihse_vector_db_open failed at %d\n", __LINE__); qihse_vector_db_destroy(vdb); errno = ENOENT; return NULL; } if (use_mmap && has_wal) { - printf("DEBUG: qihse_vector_db_open failed at %d\n", __LINE__); qihse_vector_db_destroy(vdb); errno = ENOTSUP; return NULL; } } if (!qihse_vdb_replay_wal(vdb)) { - printf("DEBUG: qihse_vector_db_open failed at %d\n", __LINE__); qihse_vector_db_destroy(vdb); return NULL; } if (vdb->wal_records_replayed != 0u) { if (!qihse_vdb_rebuild_idmap(vdb, !read_only)) { - printf("DEBUG: qihse_vector_db_open failed at %d\n", __LINE__); qihse_vector_db_destroy(vdb); return NULL; }