diff --git a/Makefile b/Makefile index 2d2dea65..0ac17aed 100644 --- a/Makefile +++ b/Makefile @@ -508,6 +508,11 @@ $(BINDIR)/$(UNITDIR)/task_system_test: $(COMMON_TESTOBJ) $(OBJDIR)/$(FUNCTIONAL_TESTSDIR)/test_async.o \ $(LIBDIR)/libsplinterdb.so +$(BINDIR)/$(UNITDIR)/superblock_test: $(COMMON_TESTOBJ) \ + $(COMMON_UNIT_TESTOBJ) \ + $(OBJDIR)/$(FUNCTIONAL_TESTSDIR)/test_async.o \ + $(LIBDIR)/libsplinterdb.so + $(BINDIR)/$(UNITDIR)/platform_apis_test: $(UTIL_SYS) \ $(COMMON_UNIT_TESTOBJ) \ $(OBJDIR)/$(TESTS_DIR)/config.o \ @@ -551,6 +556,7 @@ unit/writable_buffer_test: $(BINDIR)/$(UNITDIR)/writable_buffer_test unit/config_parse_test: $(BINDIR)/$(UNITDIR)/config_parse_test unit/limitations_test: $(BINDIR)/$(UNITDIR)/limitations_test unit/task_system_test: $(BINDIR)/$(UNITDIR)/task_system_test +unit/superblock_test: $(BINDIR)/$(UNITDIR)/superblock_test unit/splinter_shmem_test: $(BINDIR)/$(UNITDIR)/splinter_shmem_test unit/splinter_ipc_test: $(BINDIR)/$(UNITDIR)/splinter_ipc_test unit_test: $(BINDIR)/unit_test diff --git a/include/splinterdb/splinterdb.h b/include/splinterdb/splinterdb.h index abb36455..d9be5a08 100644 --- a/include/splinterdb/splinterdb.h +++ b/include/splinterdb/splinterdb.h @@ -101,6 +101,22 @@ typedef struct splinterdb_config { // log _Bool use_log; + // Automatic checkpoints: once the write-ahead log has grown by this many + // bytes, SplinterDB takes a checkpoint, which folds the logged updates into + // the durable tree and reclaims that log's space. This bounds both how much + // log a crash has to replay and how much space the log occupies. + // + // The trigger is sized in log bytes rather than in updates because the two + // are independent: a workload that repeatedly overwrites the same keys grows + // the log with every write while barely growing the tree. + // + // Zero selects a default of one cache's worth of log, on the reasoning that + // replaying much more log than the cache can hold has little to gain. To + // effectively disable automatic checkpoints -- for an application that calls + // for them itself, and accepts that the log grows until it does -- set this + // very large (UINT64_MAX). + uint64 checkpoint_log_size_bytes; + // splinter uint64 memtable_capacity; uint64 fanout; diff --git a/src/allocator.h b/src/allocator.h index eb83b631..94b7299b 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -136,18 +136,36 @@ typedef platform_status (*alloc_fn)(allocator *al, typedef refcount (*dec_ref_fn)(allocator *al, uint64 addr, page_type type); typedef refcount (*generic_ref_fn)(allocator *al, uint64 addr); -typedef platform_status (*get_super_addr_fn)(allocator *al, - allocator_root_id spl_id, - uint64 *addr); -typedef platform_status (*alloc_super_addr_fn)(allocator *al, - allocator_root_id spl_id, - uint64 *addr); -typedef void (*remove_super_addr_fn)(allocator *al, allocator_root_id spl_id); +/* + * Record one logical reference to addr while rebuilding a recovery map (see + * allocator_load_refcounts() with rebuild == TRUE). The first reference to a + * given extent establishes its nonzero allocation floor; later references + * increment it normally. addr must be the base address of a non-reserved + * extent. + */ +typedef platform_status (*recovery_record_reference_fn)(allocator *al, + uint64 addr, + page_type type); + +/* + * + */ +typedef platform_status (*generic_status_allocator_fn)(allocator *al); + +/* + * Write the refcount map to its durable location and make it durable. Returns + * the base address of the persisted map in *state_addr; the caller records that + * as the superblock's allocation_state_addr only after this returns, so the + * "map is trustworthy" flag never becomes durable before the map itself. Not + * valid while a recovery rebuild is in progress. + */ +typedef platform_status (*persist_refcounts_fn)(allocator *al, + uint64 *state_addr); + typedef uint64 (*get_size_fn)(allocator *al); typedef uint64 (*base_addr_fn)(const allocator *al, uint64 addr); -typedef void (*print_fn)(allocator *al); -typedef void (*assert_fn)(allocator *al); +typedef void (*generic_void_allocator_fn)(allocator *al); /* * Define an abstract allocator interface, holding different allocation-related @@ -155,28 +173,34 @@ typedef void (*assert_fn)(allocator *al); */ typedef struct allocator_ops { allocator_get_config_fn get_config; - alloc_fn alloc; + alloc_fn alloc; generic_ref_fn inc_ref; dec_ref_fn dec_ref; generic_ref_fn get_ref; - alloc_super_addr_fn alloc_super_addr; - get_super_addr_fn get_super_addr; - remove_super_addr_fn remove_super_addr; + // After a mount, you must do either a recovery or a load + + generic_status_allocator_fn recovery_begin; + recovery_record_reference_fn recovery_record_reference; + generic_void_allocator_fn recovery_finish; + + generic_status_allocator_fn load_refcounts; + + persist_refcounts_fn persist; get_size_fn in_use; get_size_fn get_capacity; base_addr_fn extent_base_addr; - assert_fn assert_noleaks; + generic_void_allocator_fn assert_noleaks; - print_fn print_stats; - print_fn print_allocated; + generic_void_allocator_fn print_stats; + generic_void_allocator_fn print_allocated; } allocator_ops; -// To sub-class cache, make a cache your first field; +// To sub-class allocator, make an allocator your first field; struct allocator { const allocator_ops *ops; }; @@ -212,23 +236,34 @@ allocator_get_refcount(allocator *al, uint64 addr) } static inline platform_status -allocator_get_super_addr(allocator *al, allocator_root_id spl_id, uint64 *addr) +allocator_recovery_begin(allocator *al) { - return al->ops->get_super_addr(al, spl_id, addr); + return al->ops->recovery_begin(al); } static inline platform_status -allocator_alloc_super_addr(allocator *al, - allocator_root_id spl_id, - uint64 *addr) +allocator_recovery_record_reference(allocator *al, uint64 addr, page_type type) { - return al->ops->alloc_super_addr(al, spl_id, addr); + return al->ops->recovery_record_reference(al, addr, type); } static inline void -allocator_remove_super_addr(allocator *al, allocator_root_id spl_id) +allocator_recovery_finish(allocator *al) +{ + al->ops->recovery_finish(al); +} + + +static inline platform_status +allocator_load_refcounts(allocator *al) +{ + return al->ops->load_refcounts(al); +} + +static inline platform_status +allocator_persist(allocator *al, uint64 *state_addr) { - return al->ops->remove_super_addr(al, spl_id); + return al->ops->persist(al, state_addr); } static inline uint64 diff --git a/src/blob.c b/src/blob.c index b6c8e084..f8c0c375 100644 --- a/src/blob.c +++ b/src/blob.c @@ -226,7 +226,6 @@ blob_page_iterator_get_curr(blob_page_iterator *iter, iter->cc, iter->fragment.addr, TRUE, PAGE_TYPE_BLOB); } cache_lock(iter->cc, iter->page); - cache_mark_dirty(iter->cc, iter->page); } } } @@ -327,7 +326,7 @@ blob_sync(cache *cc, slice sblob) if (!SUCCESS(rc)) { break; } - cache_page_sync(cc, itor.page, FALSE, PAGE_TYPE_BLOB); + cache_page_writeback(cc, itor.page, FALSE, PAGE_TYPE_BLOB); blob_page_iterator_advance_page(&itor); } diff --git a/src/btree.c b/src/btree.c index e93b0c50..9504aba3 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1197,7 +1197,6 @@ btree_node_lock(cache *cc, // IN btree_node *node) // IN { cache_lock(cc, node->page); - cache_mark_dirty(cc, node->page); } static inline void @@ -1294,8 +1293,6 @@ btree_create(cache *cc, btree_init_hdr(cfg, root.hdr); - cache_mark_dirty(cc, root.page); - // If this btree is for a memtable then pin all pages belonging to it if (pinned) { cache_pin(cc, root.page); diff --git a/src/cache.h b/src/cache.h index bc239b7a..598df0f5 100644 --- a/src/cache.h +++ b/src/cache.h @@ -94,17 +94,6 @@ cache_config_extent_page(const cache_config *cfg, uint64 extent_addr, uint64 i) return extent_addr + i * cache_config_page_size(cfg); } -typedef void (*cache_generic_fn)(cache *cc); -typedef uint64 (*cache_generic_uint64_fn)(cache *cc); -typedef void (*page_generic_fn)(cache *cc, page_handle *page); - -typedef page_handle *(*page_alloc_fn)(cache *cc, uint64 addr, page_type type); -typedef void (*extent_discard_fn)(cache *cc, uint64 addr, page_type type); -typedef page_handle *(*page_get_fn)(cache *cc, - uint64 addr, - bool32 blocking, - page_type type); - #define PAGE_GET_ASYNC_STATE_BUFFER_SIZE (2048) typedef union page_get_async_state_payload { uint8 bytes[PAGE_GET_ASYNC_STATE_BUFFER_SIZE]; @@ -118,6 +107,11 @@ typedef struct page_get_async_state_buffer { page_get_async_state_payload payload; } page_get_async_state_buffer; +typedef void (*cache_generic_void_fn)(cache *cc); +typedef uint64 (*cache_generic_uint64_fn)(cache *cc); +typedef platform_status (*cache_generic_status_fn)(cache *cc); +typedef void (*page_generic_fn)(cache *cc, page_handle *page); + typedef void (*page_get_async_state_init_fn)(void *payload, cache *cc, uint64 addr, @@ -127,14 +121,20 @@ typedef void (*page_get_async_state_init_fn)(void *payload, typedef async_status (*page_get_async_fn)(void *payload); typedef page_handle *(*page_get_async_state_result_fn)(void *payload); +typedef page_handle *(*page_alloc_fn)(cache *cc, uint64 addr, page_type type); +typedef void (*extent_discard_fn)(cache *cc, uint64 addr, page_type type); +typedef page_handle *(*page_get_fn)(cache *cc, + uint64 addr, + bool32 blocking, + page_type type); typedef bool32 (*page_try_claim_fn)(cache *cc, page_handle *page); -typedef void (*page_sync_fn)(cache *cc, - page_handle *page, - bool32 is_blocking, - page_type type); -typedef void (*extent_sync_fn)(cache *cc, - uint64 addr, - uint64 *pages_outstanding); +typedef void (*page_writeback_fn)(cache *cc, + page_handle *page, + bool32 is_blocking, + page_type type); +typedef void (*extent_writeback_fn)(cache *cc, + uint64 addr, + uint64 *pages_outstanding); typedef void (*page_prefetch_fn)(cache *cc, uint64 addr, page_type type); typedef int (*evict_fn)(cache *cc, bool32 ignore_pinned); typedef bool32 (*page_addr_pred_fn)(cache *cc, uint64 addr); @@ -163,35 +163,36 @@ typedef struct cache_ops { page_get_async_fn page_get_async; page_get_async_state_result_fn page_get_async_result; - page_generic_fn page_unget; - page_try_claim_fn page_try_claim; - page_generic_fn page_unclaim; - page_generic_fn page_lock; - page_generic_fn page_unlock; - page_prefetch_fn page_prefetch; - page_prefetch_fn page_prefetch_page; - page_generic_fn page_mark_dirty; - page_generic_fn page_pin; - page_generic_fn page_unpin; - page_sync_fn page_sync; - extent_sync_fn extent_sync; - cache_generic_fn flush; - evict_fn evict; - cache_generic_fn cleanup; - page_addr_pred_fn in_use; - page_addr_fn assert_ungot; - cache_generic_fn assert_free; - validate_page_fn validate_page; - cache_present_fn cache_present; - cache_print_fn print; - cache_print_fn print_stats; - io_stats_fn io_stats; - cache_generic_fn reset_stats; - count_dirty_fn count_dirty; - page_get_read_ref_fn page_get_read_ref; - enable_sync_get_fn enable_sync_get; - get_allocator_fn get_allocator; - cache_config_fn get_config; + page_generic_fn page_unget; + page_try_claim_fn page_try_claim; + page_generic_fn page_unclaim; + page_generic_fn page_lock; + page_generic_fn page_unlock; + page_prefetch_fn page_prefetch; + page_prefetch_fn page_prefetch_page; + page_generic_fn page_pin; + page_generic_fn page_unpin; + page_writeback_fn page_writeback; + extent_writeback_fn extent_writeback; + cache_generic_void_fn flush; + cache_generic_status_fn writeback_dirty; + cache_generic_status_fn durable_barrier; + evict_fn evict; + cache_generic_void_fn cleanup; + page_addr_pred_fn in_use; + page_addr_fn assert_ungot; + cache_generic_void_fn assert_free; + validate_page_fn validate_page; + cache_present_fn cache_present; + cache_print_fn print; + cache_print_fn print_stats; + io_stats_fn io_stats; + cache_generic_void_fn reset_stats; + count_dirty_fn count_dirty; + page_get_read_ref_fn page_get_read_ref; + enable_sync_get_fn enable_sync_get; + get_allocator_fn get_allocator; + cache_config_fn get_config; } cache_ops; // To sub-class cache, make a cache your first field; @@ -366,8 +367,7 @@ cache_unclaim(cache *cc, page_handle *page) * * Blocks until outstanding read locks are released by other threads. * - * If you call this method, you almost certainly want to call - * cache_mark_dirty() immediately afterward. + * Acquiring the write lock marks the page dirty. *---------------------------------------------------------------------- */ static inline void @@ -427,27 +427,6 @@ cache_prefetch_page(cache *cc, uint64 addr, page_type type) return cc->ops->page_prefetch_page(cc, addr, type); } -/* - *---------------------------------------------------------------------- - * cache_mark_dirty - * - * Marks a page changed, to be written back. - * The caller had better have the write lock on the page via cache_lock() - * before changing its value. - * - * TODO This method should be removed; its effect should come automatically - * with the acquisition of a write lock. @robj reports lots of bugs - * due to forgetting to call this method. And we can't think of a case - * where we'd want the "optimization" of taking a write lock but then - * decide not to dirty it. - *---------------------------------------------------------------------- - */ -static inline void -cache_mark_dirty(cache *cc, page_handle *page) -{ - return cc->ops->page_mark_dirty(cc, page); -} - /* *---------------------------------------------------------------------- * cache_pin @@ -484,37 +463,44 @@ cache_unpin(cache *cc, page_handle *page) /* *----------------------------------------------------------------------------- - * cache_page_sync + * cache_page_writeback * - * Asynchronously writes the page back to disk. + * Issues writeback of the page to disk. This does NOT make the page durable; + * it only hands the write to the I/O layer (no device-cache flush). * - * This is used to sync log pages opportunistically. The current API doesn't - * inform the user when this happens. "It's not the ideal API." -- @aconway + * With is_blocking == FALSE the writeback is issued asynchronously and this + * returns without waiting for completion; there is no per-call way to observe + * when it finishes. With is_blocking == TRUE the page is written synchronously + * and the write has completed by the time this returns. *----------------------------------------------------------------------------- */ static inline void -cache_page_sync(cache *cc, - page_handle *page, - bool32 is_blocking, - page_type type) +cache_page_writeback(cache *cc, + page_handle *page, + bool32 is_blocking, + page_type type) { - return cc->ops->page_sync(cc, page, is_blocking, type); + return cc->ops->page_writeback(cc, page, is_blocking, type); } /* *----------------------------------------------------------------------------- - * cache_extent_sync + * cache_extent_writeback * - * Asynchronously syncs the extent beginning at addr. + * Asynchronously issues writeback of the extent beginning at addr. This does + * NOT make the extent durable; it only hands the writes to the I/O layer (no + * device-cache flush) and returns without waiting for completion. * * *pages_outstanding is immediately incremented by the number of pages * issued for writeback (the non-clean pages of the extent); as writebacks - * complete, *pages_outstanding is decremented atomically. + * complete, *pages_outstanding is decremented atomically. This counter is the + * only way to observe when the issued writebacks finish. * * Assumes pages_outstanding is an aligned uint64, so (on x86) the caller * can access it and observe its value atomically. * - * TODO: What happens if two callers call cache_extent_sync on the same extent? + * TODO: What happens if two callers call cache_extent_writeback on the same + * extent? * * All pages in the extent must be clean or cleanable. * The page may not be in writeback, loading, or locked, or claimed, otherwise @@ -522,9 +508,9 @@ cache_page_sync(cache *cc, *----------------------------------------------------------------------------- */ static inline void -cache_extent_sync(cache *cc, uint64 addr, uint64 *pages_outstanding) +cache_extent_writeback(cache *cc, uint64 addr, uint64 *pages_outstanding) { - cc->ops->extent_sync(cc, addr, pages_outstanding); + cc->ops->extent_writeback(cc, addr, pages_outstanding); } /* @@ -542,6 +528,35 @@ cache_flush(cache *cc) cc->ops->flush(cc); } +/* + *----------------------------------------------------------------------------- + * cache_writeback_dirty + * + * Issues and wait for completion of writebacks for all pages that are dirty + * but not locked at the time of the call. May writeback other pages, as well. + *----------------------------------------------------------------------------- + */ +static inline platform_status +cache_writeback_dirty(cache *cc) +{ + return cc->ops->writeback_dirty(cc); +} + +/* + *----------------------------------------------------------------------------- + * cache_durable_barrier + * + * Ensure that writeback completed before this call is durable across a power + * loss. Callers normally use this after cache_writeback_dirty(), and again + * after publishing a checkpoint superblock. + *----------------------------------------------------------------------------- + */ +static inline platform_status +cache_durable_barrier(cache *cc) +{ + return cc->ops->durable_barrier(cc); +} + /* *----------------------------------------------------------------------------- * cache_evict diff --git a/src/clockcache.c b/src/clockcache.c index b8197236..bd59647b 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -222,6 +222,50 @@ clockcache_test_flag(clockcache *cc, uint32 entry_number, entry_status flag) return flag & clockcache_get_status(cc, entry_number); } +/* + *-------------------------------------------------------------------------- + * Dirty-generation bookkeeping + * + * Each entry records the generation in which its current dirty interval began + * (0 when clean/free). A writeback fence takes a "cut" of the generation + * counter and drains every entry stamped at or below that cut. These + * transitions need no dedicated lock: clockcache_dirty_begin runs under the + * page's write lock, and a write lock cannot be held while CC_WRITEBACK is set + * (see clockcache_get_write), so a clean->dirty transition never races a + * writeback completion on the same entry. + *-------------------------------------------------------------------------- + */ +static void +clockcache_dirty_begin(clockcache *cc, uint32 entry_number) +{ + if (clockcache_test_flag(cc, entry_number, CC_CLEAN)) { + clockcache_entry *entry = clockcache_get_entry(cc, entry_number); + debug_assert(entry->dirty_generation == 0); + // Stamp before clearing CC_CLEAN so a fence that observes the page dirty + // always sees a valid generation. clear_flag is a full barrier. + entry->dirty_generation = + __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); + clockcache_clear_flag(cc, entry_number, CC_CLEAN); + } +} + +/* + * Mark an entry clean once its writeback has completed. CC_CLEAN must be set + * before CC_WRITEBACK is cleared: the intermediate CC_CLEAN|CC_WRITEBACK state + * is not cleanable, so no thread can start a duplicate writeback in the gap. + */ +static void +clockcache_dirty_complete_writeback(clockcache *cc, uint32 entry_number) +{ + clockcache_get_entry(cc, entry_number)->dirty_generation = 0; + debug_only uint32 was_clean = + clockcache_set_flag(cc, entry_number, CC_CLEAN); + debug_assert(!was_clean); + debug_only uint32 was_writeback = + clockcache_clear_flag(cc, entry_number, CC_WRITEBACK); + debug_assert(was_writeback); +} + #ifdef RECORD_ACQUISITION_STACKS static void clockcache_record_backtrace(clockcache *cc, uint32 entry_number) @@ -785,8 +829,8 @@ clockcache_ok_to_writeback(clockcache *cc, bool32 with_access) { uint32 status = clockcache_get_status(cc, entry_number); - return ((status == CC_CLEANABLE1_STATUS) - || (with_access && status == CC_CLEANABLE2_STATUS)); + return (status == CC_CLEANABLE1_STATUS) + || (with_access && status == CC_CLEANABLE2_STATUS); } /* @@ -797,6 +841,13 @@ clockcache_ok_to_writeback(clockcache *cc, * status must be: * -- CC_CLEANABLE1_STATUS (= 0) // dirty * -- CC_CLEANABLE2_STATUS (= 0 | CC_ACCESSED) // dirty + * + * Returns FALSE only if the page is genuinely not writeback-able (locked, + * claimed, already in writeback, clean, ...). The CC_ACCESSED bit can flip + * (a reader sets it, the clock hand clears it) between the two + * compare-and- swaps, so we retry as long as the status remains one of the + * cleanable states rather than spuriously failing on a page that stayed + * cleanable. *---------------------------------------------------------------------- */ static inline bool32 @@ -811,19 +862,27 @@ clockcache_try_set_writeback(clockcache *cc, cc->cfg->page_capacity); volatile uint32 *status = &cc->entry[entry_number].status; - if (__sync_bool_compare_and_swap( - status, CC_CLEANABLE1_STATUS, CC_WRITEBACK1_STATUS)) - { - return TRUE; - } - - if (with_access - && __sync_bool_compare_and_swap( - status, CC_CLEANABLE2_STATUS, CC_WRITEBACK2_STATUS)) - { - return TRUE; + while (TRUE) { + uint32 cur = *status; + if (cur == CC_CLEANABLE1_STATUS) { + if (__sync_bool_compare_and_swap( + status, CC_CLEANABLE1_STATUS, CC_WRITEBACK1_STATUS)) + { + return TRUE; + } + } else if (with_access && cur == CC_CLEANABLE2_STATUS) { + if (__sync_bool_compare_and_swap( + status, CC_CLEANABLE2_STATUS, CC_WRITEBACK2_STATUS)) + { + return TRUE; + } + } else { + // Not a cleanable state: the page cannot be written back right now. + return FALSE; + } + // The CAS failed because the status changed under us. If it is still + // cleanable, retry; otherwise the next iteration returns FALSE. } - return FALSE; } typedef struct async_io_state { @@ -861,8 +920,6 @@ clockcache_write_callback(void *wbs) uint32 entry_number; clockcache_entry *entry; uint64 addr; - debug_only uint32 debug_status; - for (i = 0; i < count; i++) { entry_number = clockcache_data_to_entry_number(cc, (char *)iovec[i].iov_base); @@ -876,10 +933,7 @@ clockcache_write_callback(void *wbs) entry_number, addr); - debug_status = clockcache_set_flag(cc, entry_number, CC_CLEAN); - debug_assert(!debug_status); - debug_status = clockcache_clear_flag(cc, entry_number, CC_WRITEBACK); - debug_assert(debug_status); + clockcache_dirty_complete_writeback(cc, entry_number); } if (state->outstanding_pages) { @@ -922,7 +976,7 @@ clockcache_abort_writeback_range(clockcache *cc, * they are not. *---------------------------------------------------------------------- */ -void +static platform_status clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) { uint32 entry_no, next_entry_no; @@ -932,6 +986,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) uint64 end_entry_no = start_entry_no + CC_ENTRIES_PER_BATCH; clockcache_entry *entry, *next_entry; + platform_status result = STATUS_OK; debug_assert((tid < MAX_THREADS), "Invalid tid=%lu\n", tid); debug_assert(cc != NULL); @@ -968,6 +1023,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) next_entry_no = CC_UNMAPPED_ENTRY; } while ( next_entry_no != CC_UNMAPPED_ENTRY + && clockcache_ok_to_writeback(cc, next_entry_no, is_urgent) && clockcache_try_set_writeback(cc, next_entry_no, is_urgent)); first_addr += page_size; end_addr = entry->page.disk_addr; @@ -981,6 +1037,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) next_entry_no = CC_UNMAPPED_ENTRY; } while ( next_entry_no != CC_UNMAPPED_ENTRY + && clockcache_ok_to_writeback(cc, next_entry_no, is_urgent) && clockcache_try_set_writeback(cc, next_entry_no, is_urgent)); @@ -991,6 +1048,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) "clockcache_batch_start_writeback: async_io_state allocation " "failed\n"); clockcache_abort_writeback_range(cc, first_addr, end_addr); + result = STATUS_NO_MEMORY; goto close_log; } @@ -1008,6 +1066,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) platform_status_to_string(rc)); clockcache_abort_writeback_range(cc, first_addr, end_addr); platform_free(PROCESS_PRIVATE_HEAP_ID, state); + result = rc; goto close_log; } @@ -1033,6 +1092,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) io_async_state_deinit(state->iostate); clockcache_abort_writeback_range(cc, first_addr, end_addr); platform_free(PROCESS_PRIVATE_HEAP_ID, state); + result = rc; goto close_log; } } @@ -1042,12 +1102,95 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) cc->stats[tid].writes_issued++; } - io_async_run(state->iostate); + // The IO layer must run writeback asynchronously and complete it via + // clockcache_write_callback (which does the dirty->clean bookkeeping). + // A synchronous completion would skip that callback and strand the + // pages in CC_WRITEBACK, so a broken contract is a fatal correctness + // error. + async_status arc = io_async_run(state->iostate); + platform_assert( + arc == ASYNC_STATUS_RUNNING, + "clockcache_batch_start_writeback: async writeback for addr %lu " + "completed synchronously; the IO layer must complete it via the " + "write callback", + first_addr); } } close_log: clockcache_close_log_stream(); + return result; +} + +/* + *-------------------------------------------------------------------------- + * clockcache_writeback_dirty -- + * + * Write back every page that was dirty when this call began, and wait for + * those writes (only those) to complete, so a subsequent durable_barrier can + * make them durable. + * + * We take a "cut" by incrementing the dirty generation: pages dirtied before + * now carry a generation <= cutoff and must be drained; pages dirtied + * afterward carry a higher generation and never delay this call. This is what + * lets us avoid io_wait_all, which waits for *global* I/O quiescence -- a + * condition a busy cache (background cleaner writes, async reads) may never + * reach. + * + * A single scan of the entries suffices, because a generation only ever moves + * above the cut (a page must go clean before it can be re-dirtied), so an entry + * that has left the pre-cut set never re-enters it. We first issue writeback + * for every dirty, unlocked page in one bulk pass so the writes pipeline, then + * walk the entries once, waiting per entry for its pre-cut interval to drain. + * + * The bulk pass leaves every pre-cut page that belongs to this checkpoint in + * CC_WRITEBACK: such a page is dirty and, under copy-on-write, never locked or + * claimed, so clockcache_try_set_writeback issues it (or a concurrent cleaner + * already has). The drain therefore only has to wait on pages that are in + * writeback; a pre-cut page that is not in writeback is either already clean, + * or was held by a writer during the bulk pass and so cannot belong to this + * checkpoint -- either way we skip it, which is what keeps this deadlock-free. + * Per-context background reapers complete the issued writes; we also poll our + * own context via clockcache_wait to help things along. + * + * Termination: once a page is in writeback the writer is excluded (see + * clockcache_get_write), so it progresses to clean and, if re-dirtied, moves to + * a generation above the cut. + *-------------------------------------------------------------------------- + */ +platform_status +clockcache_writeback_dirty(clockcache *cc) +{ + uint64 cutoff = + __atomic_fetch_add(&cc->dirty_generation, 1, __ATOMIC_SEQ_CST); + platform_assert(cutoff < UINT64_MAX); + + // Bulk-issue writeback for every dirty, unlocked page so the writes pipeline + // rather than draining one batch at a time. + for (uint64 batch = 0; batch < cc->cfg->batch_capacity; batch++) { + platform_status result = + clockcache_batch_start_writeback(cc, batch, TRUE); + if (!SUCCESS(result)) { + return result; + } + } + + // Wait for each pre-cut interval's writeback to complete, in a single pass. + for (uint32 entry_no = 0; entry_no < cc->cfg->page_capacity; entry_no++) { + clockcache_entry *entry = clockcache_get_entry(cc, entry_no); + while (TRUE) { + uint64 gen = + __atomic_load_n(&entry->dirty_generation, __ATOMIC_RELAXED); + if (gen == 0 || gen > cutoff + || !clockcache_test_flag(cc, entry_no, CC_WRITEBACK)) + { + break; + } + clockcache_wait(cc); + } + } + + return STATUS_OK; } /* @@ -1147,8 +1290,9 @@ clockcache_try_evict(clockcache *cc, uint32 entry_number) /* 6. set status to CC_FREE_STATUS (clears claim and write lock) */ platform_assert(entry->waiters.head == NULL); - entry->type = PAGE_TYPE_INVALID; - entry->status = CC_FREE_STATUS; + entry->type = PAGE_TYPE_INVALID; + entry->dirty_generation = 0; + entry->status = CC_FREE_STATUS; clockcache_log( addr, entry_number, "evict: entry %u addr %lu\n", entry_number, addr); @@ -1228,7 +1372,12 @@ clockcache_move_hand(clockcache *cc, bool32 is_urgent) cleaner_hand = (evict_hand + cc->cleaner_gap) % cc->cfg->batch_capacity; clean_batch_busy = &cc->batch_busy[cleaner_hand]; if (__sync_bool_compare_and_swap(clean_batch_busy, FALSE, TRUE)) { - clockcache_batch_start_writeback(cc, cleaner_hand, is_urgent); + platform_status rc = + clockcache_batch_start_writeback(cc, cleaner_hand, is_urgent); + if (!SUCCESS(rc)) { + platform_error_log("clockcache_move_hand: writeback failed: %s\n", + platform_status_to_string(rc)); + } was_busy = __sync_bool_compare_and_swap(clean_batch_busy, TRUE, FALSE); debug_assert(was_busy); } @@ -1279,12 +1428,22 @@ clockcache_get_free_page(clockcache *cc, && __sync_bool_compare_and_swap( &entry->status, CC_FREE_STATUS, CC_ALLOC_STATUS)) { + + // A page that begins dirty (a fresh allocation) must carry a dirty + // generation, just like a clean->dirty transition. The entry is + // write-locked here, so a concurrent fence skips it regardless. + if (!(status & CC_CLEAN)) { + debug_assert(entry->dirty_generation == 0); + entry->dirty_generation = + __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); + } + entry->status = status; + entry->type = type; + if (refcount) { clockcache_inc_ref(cc, entry_no, tid); } platform_assert(entry->waiters.head == NULL); - entry->status = status; - entry->type = type; debug_assert(entry->page.disk_addr == CC_UNMAPPED_ADDR); clockcache_record_backtrace(cc, entry_no); return entry_no; @@ -1343,7 +1502,9 @@ clockcache_flush(clockcache *cc) flush_hand < cc->cfg->page_capacity / CC_ENTRIES_PER_BATCH; flush_hand++) { - clockcache_batch_start_writeback(cc, flush_hand, TRUE); + platform_status rc = + clockcache_batch_start_writeback(cc, flush_hand, TRUE); + platform_assert_status_ok(rc); } // make sure all aio is complete again @@ -1503,8 +1664,9 @@ clockcache_try_page_discard(clockcache *cc, uint64 addr) /* 6. set status to CC_FREE_STATUS (clears claim and write lock) */ platform_assert(entry->waiters.head == NULL); - entry->type = PAGE_TYPE_INVALID; - entry->status = CC_FREE_STATUS; + entry->type = PAGE_TYPE_INVALID; + entry->dirty_generation = 0; + entry->status = CC_FREE_STATUS; /* 7. reset pincount */ clockcache_reset_pin(cc, entry_number); @@ -1621,8 +1783,9 @@ clockcache_get_in_cache(clockcache *cc, // IN static void clockcache_release_unpublished_entry(clockcache_entry *entry) { - entry->page.disk_addr = CC_UNMAPPED_ADDR; - entry->type = PAGE_TYPE_INVALID; + entry->page.disk_addr = CC_UNMAPPED_ADDR; + entry->type = PAGE_TYPE_INVALID; + entry->dirty_generation = 0; platform_assert(entry->waiters.head == NULL); entry->status = CC_FREE_STATUS; } @@ -2166,6 +2329,10 @@ clockcache_lock(clockcache *cc, page_handle *page) entry_number, page->disk_addr); clockcache_get_write(cc, entry_number); + // A write lock marks the page dirty (and stamps its dirty generation). The + // CC_WRITEBACK exclusion in clockcache_get_write guarantees this cannot race + // a writeback completion. + clockcache_dirty_begin(cc, entry_number); } void @@ -2185,27 +2352,6 @@ clockcache_unlock(clockcache *cc, page_handle *page) } -/*---------------------------------------------------------------------- - * clockcache_mark_dirty -- - * - * Marks the entry dirty. - *---------------------------------------------------------------------- - */ -void -clockcache_mark_dirty(clockcache *cc, page_handle *page) -{ - debug_only clockcache_entry *entry = clockcache_page_to_entry(cc, page); - uint32 entry_number = clockcache_page_to_entry_number(cc, page); - - clockcache_log(entry->page.disk_addr, - entry_number, - "mark_dirty: entry %u addr %lu\n", - entry_number, - entry->page.disk_addr); - clockcache_clear_flag(cc, entry_number, CC_CLEAN); - return; -} - /* *---------------------------------------------------------------------- * clockcache_pin -- @@ -2248,17 +2394,22 @@ clockcache_unpin(clockcache *cc, page_handle *page) /* *----------------------------------------------------------------------------- - * clockcache_page_sync -- + * clockcache_page_writeback -- * - * Asynchronously syncs the page. Currently there is no way to check - *when the writeback has completed. + * Issues writeback of the page. This does not make the page durable; it + * only hands the write to the I/O layer. + * + * With is_blocking == FALSE the writeback is issued asynchronously and + * this returns without waiting for completion. With is_blocking == TRUE + * the page is written synchronously and has completed by the time this + * returns. *----------------------------------------------------------------------------- */ void -clockcache_page_sync(clockcache *cc, - page_handle *page, - bool32 is_blocking, - page_type type) +clockcache_page_writeback(clockcache *cc, + page_handle *page, + bool32 is_blocking, + page_type type) { uint32 entry_number = clockcache_page_to_entry_number(cc, page); async_io_state *state; @@ -2266,9 +2417,26 @@ clockcache_page_sync(clockcache *cc, const threadid tid = platform_get_tid(); platform_status status; - if (!clockcache_try_set_writeback(cc, entry_number, TRUE)) { - platform_assert(clockcache_test_flag(cc, entry_number, CC_CLEAN)); - return; + while (!clockcache_try_set_writeback(cc, entry_number, TRUE)) { + if (clockcache_test_flag(cc, entry_number, CC_CLEAN)) { + return; + } + + /* + * A pressure cleaner or a checkpoint fence may have begun writeback + * after the caller released its claim. Wait for that writeback rather + * than treating a perfectly valid concurrent flush as an assertion. + */ + if (clockcache_test_flag(cc, entry_number, CC_WRITEBACK)) { + clockcache_wait(cc); + continue; + } + + platform_assert(0, + "page_writeback requires a cleanable page: entry=%u " + "status=%u\n", + entry_number, + clockcache_get_status(cc, entry_number)); } if (cc->cfg->use_stats) { @@ -2279,11 +2447,12 @@ clockcache_page_sync(clockcache *cc, if (!is_blocking) { state = TYPED_MALLOC(PROCESS_PRIVATE_HEAP_ID, state); if (state == NULL) { - platform_error_log("clockcache_page_sync: async_io_state allocation " - "failed for addr %lu, entry %u, type %u\n", - addr, - entry_number, - type); + platform_error_log( + "clockcache_page_writeback: async_io_state allocation " + "failed for addr %lu, entry %u, type %u\n", + addr, + entry_number, + type); } platform_assert(state); state->cc = cc; @@ -2295,65 +2464,68 @@ clockcache_page_sync(clockcache *cc, clockcache_write_callback, state); if (!SUCCESS(status)) { - platform_error_log("clockcache_page_sync: io_async_state_init failed " - "for addr %lu, entry %u, type %u: %s\n", - addr, - entry_number, - type, - platform_status_to_string(status)); + platform_error_log( + "clockcache_page_writeback: io_async_state_init failed " + "for addr %lu, entry %u, type %u: %s\n", + addr, + entry_number, + type, + platform_status_to_string(status)); } platform_assert_status_ok(status); status = io_async_state_append_page(state->iostate, page->data); if (!SUCCESS(status)) { - platform_error_log("clockcache_page_sync: io_async_state_append_page " - "failed for addr %lu, entry %u, type %u: %s\n", - addr, - entry_number, - type, - platform_status_to_string(status)); + platform_error_log( + "clockcache_page_writeback: io_async_state_append_page " + "failed for addr %lu, entry %u, type %u: %s\n", + addr, + entry_number, + type, + platform_status_to_string(status)); } platform_assert_status_ok(status); io_async_run(state->iostate); } else { status = io_write(cc->io, page->data, clockcache_page_size(cc), addr); if (!SUCCESS(status)) { - platform_error_log("clockcache_page_sync: io_write failed for addr " - "%lu, entry %u, type %u: %s\n", - addr, - entry_number, - type, - platform_status_to_string(status)); + platform_error_log( + "clockcache_page_writeback: io_write failed for addr " + "%lu, entry %u, type %u: %s\n", + addr, + entry_number, + type, + platform_status_to_string(status)); } platform_assert_status_ok(status); clockcache_log(addr, entry_number, - "page_sync write entry %u addr %lu\n", + "page_writeback write entry %u addr %lu\n", entry_number, addr); - debug_only uint8 rc; - rc = clockcache_set_flag(cc, entry_number, CC_CLEAN); - debug_assert(!rc); - rc = clockcache_clear_flag(cc, entry_number, CC_WRITEBACK); - debug_assert(rc); + clockcache_dirty_complete_writeback(cc, entry_number); } } /* *----------------------------------------------------------------------------- - * clockcache_extent_sync -- + * clockcache_extent_writeback -- * - * Asynchronously syncs the extent. + * Asynchronously issues writeback of the extent. This does not make the + * extent durable; it only hands the writes to the I/O layer and returns + * without waiting for completion. * * Adds the number of pages issued writeback to the counter pointed to * by pages_outstanding. When the writes complete, a callback subtracts * them off, so that the caller may track how many pages are in - *writeback. + * writeback. * * Assumes all pages in the extent are clean or cleanable *----------------------------------------------------------------------------- */ void -clockcache_extent_sync(clockcache *cc, uint64 addr, uint64 *pages_outstanding) +clockcache_extent_writeback(clockcache *cc, + uint64 addr, + uint64 *pages_outstanding) { async_io_state *state = NULL; uint64 i; @@ -2372,7 +2544,7 @@ clockcache_extent_sync(clockcache *cc, uint64 addr, uint64 *pages_outstanding) req_addr = page_addr; state = TYPED_MALLOC(PROCESS_PRIVATE_HEAP_ID, state); if (state == NULL) { - platform_error_log("clockcache_extent_sync: async_io_state " + platform_error_log("clockcache_extent_writeback: async_io_state " "allocation failed for extent addr %lu, " "page addr %lu, entry %u\n", addr, @@ -2389,7 +2561,7 @@ clockcache_extent_sync(clockcache *cc, uint64 addr, uint64 *pages_outstanding) clockcache_write_callback, state); if (!SUCCESS(rc)) { - platform_error_log("clockcache_extent_sync: " + platform_error_log("clockcache_extent_writeback: " "io_async_state_init failed for extent addr " "%lu, req addr %lu, entry %u: %s\n", addr, @@ -2402,7 +2574,7 @@ clockcache_extent_sync(clockcache *cc, uint64 addr, uint64 *pages_outstanding) platform_status rc = io_async_state_append_page( state->iostate, clockcache_get_entry(cc, entry_number)->page.data); if (!SUCCESS(rc)) { - platform_error_log("clockcache_extent_sync: " + platform_error_log("clockcache_extent_writeback: " "io_async_state_append_page failed for extent " "addr %lu, page addr %lu, entry %u: %s\n", addr, @@ -3095,13 +3267,6 @@ clockcache_prefetch_page_virtual(cache *c, uint64 addr, page_type type) clockcache_prefetch_page(cc, addr, type); } -void -clockcache_mark_dirty_virtual(cache *c, page_handle *page) -{ - clockcache *cc = (clockcache *)c; - clockcache_mark_dirty(cc, page); -} - void clockcache_pin_virtual(cache *c, page_handle *page) { @@ -3148,20 +3313,22 @@ clockcache_get_async_state_result_virtual(void *payload) } void -clockcache_page_sync_virtual(cache *c, - page_handle *page, - bool32 is_blocking, - page_type type) +clockcache_page_writeback_virtual(cache *c, + page_handle *page, + bool32 is_blocking, + page_type type) { clockcache *cc = (clockcache *)c; - clockcache_page_sync(cc, page, is_blocking, type); + clockcache_page_writeback(cc, page, is_blocking, type); } void -clockcache_extent_sync_virtual(cache *c, uint64 addr, uint64 *pages_outstanding) +clockcache_extent_writeback_virtual(cache *c, + uint64 addr, + uint64 *pages_outstanding) { clockcache *cc = (clockcache *)c; - clockcache_extent_sync(cc, addr, pages_outstanding); + clockcache_extent_writeback(cc, addr, pages_outstanding); } void @@ -3171,6 +3338,20 @@ clockcache_flush_virtual(cache *c) clockcache_flush(cc); } +platform_status +clockcache_writeback_dirty_virtual(cache *c) +{ + clockcache *cc = (clockcache *)c; + return clockcache_writeback_dirty(cc); +} + +platform_status +clockcache_durable_barrier_virtual(cache *c) +{ + clockcache *cc = (clockcache *)c; + return io_durable_barrier(cc->io); +} + int clockcache_evict_all_virtual(cache *c, bool32 ignore_pinned) { @@ -3299,12 +3480,13 @@ static cache_ops clockcache_ops = { .page_unlock = clockcache_unlock_virtual, .page_prefetch = clockcache_prefetch_virtual, .page_prefetch_page = clockcache_prefetch_page_virtual, - .page_mark_dirty = clockcache_mark_dirty_virtual, .page_pin = clockcache_pin_virtual, .page_unpin = clockcache_unpin_virtual, - .page_sync = clockcache_page_sync_virtual, - .extent_sync = clockcache_extent_sync_virtual, + .page_writeback = clockcache_page_writeback_virtual, + .extent_writeback = clockcache_extent_writeback_virtual, .flush = clockcache_flush_virtual, + .writeback_dirty = clockcache_writeback_dirty_virtual, + .durable_barrier = clockcache_durable_barrier_virtual, .evict = clockcache_evict_all_virtual, .cleanup = clockcache_wait_virtual, .in_use = clockcache_in_use_virtual, @@ -3397,6 +3579,10 @@ clockcache_init(clockcache *cc, // OUT cc->io = io; cc->heap_id = hid; + // Generation 0 is the sentinel for "clean/free", so dirty stamping starts + // at 1. + cc->dirty_generation = 1; + /* lookup maps addrs to entries, entry contains the entries themselves */ platform_status rc = platform_buffer_init( &cc->lookup_bh, allocator_page_capacity * sizeof(cc->lookup[0])); @@ -3438,9 +3624,10 @@ clockcache_init(clockcache *cc, // OUT for (i = 0; i < cc->cfg->page_capacity; i++) { cc->entry[i].page.data = cc->data + clockcache_multiply_by_page_size(cc, i); - cc->entry[i].page.disk_addr = CC_UNMAPPED_ADDR; - cc->entry[i].status = CC_FREE_STATUS; - cc->entry[i].type = PAGE_TYPE_INVALID; + cc->entry[i].page.disk_addr = CC_UNMAPPED_ADDR; + cc->entry[i].status = CC_FREE_STATUS; + cc->entry[i].dirty_generation = 0; + cc->entry[i].type = PAGE_TYPE_INVALID; async_wait_queue_init(&cc->entry[i].waiters); } diff --git a/src/clockcache.h b/src/clockcache.h index 814cde6a..cb5929fd 100644 --- a/src/clockcache.h +++ b/src/clockcache.h @@ -72,8 +72,13 @@ typedef uint32 entry_status; // Saved in clockcache_entry->status struct clockcache_entry { page_handle page; volatile entry_status status; - page_type type; - async_wait_queue waiters; + // Generation in which this page's current dirty interval began; 0 when the + // page is clean or free. A writeback_dirty() drains every entry whose + // generation is at or below the cut it took (see + // clockcache_writeback_dirty). + volatile uint64 dirty_generation; + page_type type; + async_wait_queue waiters; #ifdef RECORD_ACQUISITION_STACKS int next_history_record; history_record history[NUM_HISTORY_RECORDS]; @@ -139,6 +144,11 @@ struct clockcache { volatile bool32 *batch_busy; // Convenience pointer for batch_bh uint64 cleaner_gap; + // Monotonic generation counter. A writeback_dirty() atomically increments it + // to take a "cut", then drains every entry stamped with a generation at or + // below that cut. + uint64 dirty_generation; + volatile struct { volatile uint32 free_hand; bool32 enable_sync_get; diff --git a/src/core.c b/src/core.c index d499f24a..84134b93 100644 --- a/src/core.c +++ b/src/core.c @@ -8,6 +8,7 @@ */ #include "core.h" +#include "shard_log.h" // core constructs the concrete log via shard_log_create() #include "data_internal.h" #include "notification.h" #include "platform_sleep.h" @@ -46,8 +47,42 @@ static const int64 latency_histo_buckets[LATENCYHISTO_SIZE] = { _Static_assert(CORE_NUM_MEMTABLES <= MAX_MEMTABLES, "CORE_NUM_MEMTABLES <= MAX_MEMTABLES"); -/* Some randomly chosen Splinter super-block checksum seed. */ -#define CORE_SUPER_CSUM_SEED (42) +/* + * Initialize the instance's two locks -- superblock_lock (publication) and + * checkpoint_state_lock (the checkpoint phase machine) -- plus the checkpoint + * state they guard. Transactional: on failure nothing is left initialized, so + * callers just bail out and must not call core_locks_deinit(). + */ +static platform_status +core_locks_init(core_handle *spl) +{ + platform_status rc = platform_mutex_init( + &spl->superblock_lock, platform_get_module_id(), spl->heap_id); + if (!SUCCESS(rc)) { + return rc; + } + + rc = platform_mutex_init( + &spl->checkpoint_state_lock, platform_get_module_id(), spl->heap_id); + if (!SUCCESS(rc)) { + platform_status destroy_rc = + platform_mutex_destroy(&spl->superblock_lock); + platform_assert_status_ok(destroy_rc); + return rc; + } + + ZERO_CONTENTS(&spl->checkpoint); // phase == CORE_CHECKPOINT_IDLE + return STATUS_OK; +} + +static void +core_locks_deinit(core_handle *spl) +{ + platform_status rc = platform_mutex_destroy(&spl->checkpoint_state_lock); + platform_assert_status_ok(rc); + rc = platform_mutex_destroy(&spl->superblock_lock); + platform_assert_status_ok(rc); +} /* * core logging functions. @@ -107,142 +142,545 @@ core_close_log_stream_if_enabled(core_handle *spl, /* *----------------------------------------------------------------------------- - * Splinter Super Block: Disk-resident structure. - * Super block lives on page of page type == PAGE_TYPE_SUPERBLOCK. + * Checkpoint record functions *----------------------------------------------------------------------------- */ -typedef struct ONDISK core_super_block { - uint64 root_addr; // Address of the root of the trunk for the instance - // referenced by this superblock. - uint64 log_addr; - uint64 log_meta_addr; - uint64 timestamp; - bool32 checkpointed; - bool32 unmounted; - checksum128 checksum; -} core_super_block; +static platform_status +core_checkpoint_capture_cut(core_handle *spl, + trunk_snapshot *snapshot, + uint64 *first_unincorporated_generation) +{ + /* + * Incorporation publishes its generation and root while holding lookup + * exclusion before taking the trunk root lock. Take the checkpoint cut + * in the same order, so a record never combines a pre-incorporation + * generation with a post-incorporation root (or the converse). + */ + memtable_block_lookups(&spl->mt_ctxt); + uint64 retired_generation = memtable_generation_retired(&spl->mt_ctxt); + platform_status rc = trunk_snapshot_create(&spl->trunk_context, snapshot); + memtable_unblock_lookups(&spl->mt_ctxt); + if (!SUCCESS(rc)) { + return rc; + } + + /* + * The first generation not folded into the root -- the exclusive replay + * bound. When nothing has been retired, memtable_generation_retired() is + * UINT64_MAX and this wraps to 0 ("replay from generation 0"), so no + * sentinel is needed. + */ + *first_unincorporated_generation = retired_generation + 1; + return STATUS_OK; +} /* - *----------------------------------------------------------------------------- - * Super block functions - *----------------------------------------------------------------------------- + * Translate the log module's log_head into the superblock's descriptor, adding + * the coverage information the log module does not track: start_generation is + * the first memtable generation whose entries went to this log. + */ +static superblock_log_head +core_log_to_superblock_log_head(log_head info, uint64 start_generation) +{ + return (superblock_log_head){.addr = info.addr, + .meta_addr = info.meta_addr, + .magic = info.magic, + .start_generation = start_generation}; +} + +/* + * Commit the trunk's current COW root as the new durable tree root: capture the + * root, make its pages durable, snapshot it into the superblock (recording the + * first unincorporated generation, and dropping the sealed log if this root now + * covers it), then release the previously published root. The log slots are + * the cut protocol's business, so this leaves the live log alone; callers that + * need to install or discard a log do so with superblock_log_cut() / + * superblock_discard_logs() before calling this, and the single publish below + * commits both transitions together. + * + * snapshot_tree invalidates the persisted allocation state; a clean unmount + * revalidates it in a later step. Used by mkfs, checkpoint completion, a + * durability checkpoint, and unmount (Part A). On success the captured + * reference becomes the durable record's and the previously published root's is + * released; republishing an unchanged root is just the degenerate case of that, + * so it needs no special handling. + * + * Note this always publishes, even when the root is unchanged: callers stage + * log transitions into the image beforehand, and the generation bound can + * advance on its own (an empty generation retires without changing the root), + * so the root address alone is not a "nothing to do" test. */ static platform_status -core_set_super_block(core_handle *spl, - bool32 is_checkpoint, - bool32 is_unmount, - bool32 is_create) -{ - uint64 super_addr; - page_handle *super_page; - core_super_block *super; - uint64 wait = 1; - platform_status rc; - - if (is_create) { - rc = allocator_alloc_super_addr(spl->al, spl->id, &super_addr); - } else { - rc = allocator_get_super_addr(spl->al, spl->id, &super_addr); - } +core_checkpoint_commit_current_root(core_handle *spl) +{ + platform_status rc; + trunk_snapshot snapshot; + uint64 first_unincorporated_generation; + uint64 old_root_addr = 0; + superblock_tree_record old_rec; + + /* + * The snapshot cut, durable record write, and old-root release are one + * publication transaction; serialize against any concurrent publisher. + */ + rc = platform_mutex_lock(&spl->superblock_lock); if (!SUCCESS(rc)) { - platform_error_log("core_set_super_block: failed to %s super block " - "address for root id %lu: %s\n", - is_create ? "allocate" : "get", - spl->id, - platform_status_to_string(rc)); return rc; } - super_page = cache_get(spl->cc, super_addr, TRUE, PAGE_TYPE_SUPERBLOCK); - while (!cache_try_claim(spl->cc, super_page)) { - platform_sleep_ns(wait); - wait *= 2; - } - wait = 1; - cache_lock(spl->cc, super_page); - super = (core_super_block *)super_page->data; - uint64 old_root_addr = super->root_addr; + rc = core_checkpoint_capture_cut( + spl, &snapshot, &first_unincorporated_generation); + if (!SUCCESS(rc)) { + goto unlock_superblock; + } - trunk_ondisk_node_handle root_handle; - trunk_init_root_handle(&spl->trunk_context, &root_handle); - uint64 root_addr = trunk_ondisk_node_handle_addr(&root_handle); - if (root_addr != 0) { - trunk_inc_ref(spl->al, root_addr); + /* + * The snapshot reference makes the root stable, but not necessarily + * durable. Drain the cache so the pages the record will name are durable + * before we publish a superblock that points at them. This can + * incidentally persist newer log/data pages, but it does not seal or + * publish a logical durable-log tail; tail sync is a separate operation. + */ + rc = cache_writeback_dirty(spl->cc); + if (!SUCCESS(rc)) { + goto release_snapshot; + } + rc = cache_durable_barrier(spl->cc); + if (!SUCCESS(rc)) { + goto release_snapshot; } - super->root_addr = root_addr; - trunk_ondisk_node_handle_deinit(&root_handle); - if (spl->cfg.use_log) { - if (spl->log) { - super->log_addr = log_addr(spl->log); - super->log_meta_addr = log_meta_addr(spl->log); - } else { - super->log_addr = 0; - super->log_meta_addr = 0; - } + // The previously published root, retained until the new one is durable. + superblock_get_tree_record(&spl->superblock, &old_rec); + old_root_addr = old_rec.root_addr; + + /* + * Snapshot the new root and make it durable. snapshot_tree invalidates the + * persisted allocation map -- the in-memory map now diverges from disk; a + * clean unmount revalidates it only after persisting the map (Part B). + */ + superblock_snapshot_tree( + &spl->superblock, snapshot.root_addr, first_unincorporated_generation); + + rc = superblock_make_durable(&spl->superblock); + if (!SUCCESS(rc)) { + /* The old root is still the newest durable one; keep its reference. */ + goto release_snapshot; } - super->timestamp = platform_get_real_time(); - super->checkpointed = is_checkpoint; - super->unmounted = is_unmount; - super->checksum = - platform_checksum128(super, - sizeof(core_super_block) - sizeof(checksum128), - CORE_SUPER_CSUM_SEED); - - cache_mark_dirty(spl->cc, super_page); - cache_unlock(spl->cc, super_page); - cache_unclaim(spl->cc, super_page); - cache_unget(spl->cc, super_page); - cache_page_sync(spl->cc, super_page, TRUE, PAGE_TYPE_SUPERBLOCK); - - if (old_root_addr != 0 && !is_create) { - rc = trunk_dec_ref(spl->cfg.trunk_node_cfg, - PROCESS_PRIVATE_HEAP_ID, - spl->cc, - spl->al, - spl->ts, - old_root_addr); - if (!SUCCESS(rc)) { - platform_error_log("core_set_super_block: trunk_dec_ref failed for " - "old root addr %lu: %s\n", + + /* + * The captured reference becomes the record's durable one, and the + * previously published root's reference is released. This is uniform even + * when the root did not change: that root's count is momentarily 2 (the + * record's plus ours) and the release brings it back to the record's single + * reference, so a same-root republish needs no special case and cannot grow + * the count. The publish barrier already committed the new root to the + * newer superblock slot, so the old slot is no longer the mount choice and + * releasing it cannot strand a torn-write fallback. + */ + snapshot.root_addr = 0; // transferred to the durable record + if (old_root_addr != 0) { + trunk_snapshot old_snapshot = {.root_addr = old_root_addr}; + platform_status release_rc = + trunk_snapshot_release(&spl->trunk_context, &old_snapshot); + if (!SUCCESS(release_rc)) { + platform_error_log("core_checkpoint_commit_current_root: " + "trunk_snapshot_release failed for old root addr " + "%lu: %s\n", old_root_addr, - platform_status_to_string(rc)); - return rc; + platform_status_to_string(release_rc)); + if (SUCCESS(rc)) { + rc = release_rc; + } } } - return STATUS_OK; + goto unlock_superblock; + +release_snapshot: +{ + platform_status release_rc = + trunk_snapshot_release(&spl->trunk_context, &snapshot); + if (SUCCESS(rc) && !SUCCESS(release_rc)) { + rc = release_rc; + } } -static core_super_block * -core_get_super_block_if_valid(core_handle *spl, page_handle **super_page) +unlock_superblock: { - uint64 super_addr; - core_super_block *super; + platform_status unlock_rc = platform_mutex_unlock(&spl->superblock_lock); + if (SUCCESS(rc) && !SUCCESS(unlock_rc)) { + rc = unlock_rc; + } +} + return rc; +} - platform_status rc = allocator_get_super_addr(spl->al, spl->id, &super_addr); - platform_assert_status_ok(rc); - *super_page = cache_get(spl->cc, super_addr, TRUE, PAGE_TYPE_SUPERBLOCK); - super = (core_super_block *)(*super_page)->data; - - if (!platform_checksum_is_equal( - super->checksum, - platform_checksum128(super, - sizeof(core_super_block) - sizeof(checksum128), - CORE_SUPER_CSUM_SEED))) - { - cache_unget(spl->cc, *super_page); - *super_page = NULL; - return NULL; +/* + *----------------------------------------------------------------------------- + * Incorporation-driven checkpoint (two-log protocol) + * + * A checkpoint rotates the log and advances the durable root without stopping + * the world. It is driven off memtable rotation and incorporation: + * + * begin (rotation): pre-create the next live log outside the critical + * section, swap it in under the insert lock (so no writer can be + * mid-write to the old log), then seal the old log just after. + * complete (incorporation): once the sealed log's generations are folded + * into the trunk root, publish the advanced root with the sealed + * slot cleared and free the sealed log's extents. + * + * See core_checkpoint_state in core.h for the phase machine and its locking. + *----------------------------------------------------------------------------- + */ + +/* + * Policy: should the next memtable rotation start a checkpoint? Triggered by + * the live log's size, since that is what a checkpoint reclaims -- and it is + * the only measure that tracks a workload which overwrites in place, filling + * the log without ever filling a memtable. + * + * Callers hold checkpoint_state_lock, which is also held while spl->log is + * swapped, so the log read below cannot race the cut. + */ +static bool32 +core_should_take_checkpoint(core_handle *spl) +{ + if (!spl->cfg.use_log || spl->cfg.checkpoint_log_size_bytes == 0) { + return FALSE; + } + /* + * log_get_size() reports bytes *appended*, so the fresh log a cut installs + * reads 0 and this settles rather than rotating in a loop. + */ + return log_get_size(spl->log) >= spl->cfg.checkpoint_log_size_bytes; +} + +/* + * A consistent read of what a waiter needs to poll: the current phase (has the + * cut happened yet?) and the completion count (has a given checkpoint finished + * and freed its log?). Taken together under one acquisition of the state lock. + */ +typedef struct core_checkpoint_status { + core_checkpoint_phase phase; + uint64 completions; +} core_checkpoint_status; + +static core_checkpoint_status +core_checkpoint_status_get(core_handle *spl) +{ + platform_mutex_lock(&spl->checkpoint_state_lock); + core_checkpoint_status status = {.phase = spl->checkpoint.phase, + .completions = spl->checkpoint.completions}; + platform_mutex_unlock(&spl->checkpoint_state_lock); + return status; +} + +/* + * Begin, step 1 (outside the rotation critical section): if no checkpoint is in + * progress, and either the caller forces it or the interval policy says so, + * pre-create the next live log and arm the swap. Log creation does no disk + * I/O, but is kept off the insert-blocking path. + * + * `force` bypasses only the interval policy, never the use_log precondition: + * without a log there is nothing to cut, and arming would leave the rotate hook + * dereferencing a NULL spl->log. + * + * Returns a completion ticket for the checkpoint this call armed: it has + * finished, and freed its retired log, once checkpoint.completions reaches the + * ticket. The ticket is captured under the same lock acquisition that arms, so + * it cannot miss or over-count a completion. Returns 0 if this call did not + * arm one, which is not an error -- only one checkpoint can be in flight at a + * time, and declining is the normal outcome when one already is. Tickets are + * 1-based, so 0 is unambiguous. + */ +static uint64 +core_checkpoint_begin(core_handle *spl, bool32 force) +{ + if (!spl->cfg.use_log) { + return 0; + } + + platform_mutex_lock(&spl->checkpoint_state_lock); + bool32 begin = spl->checkpoint.phase == CORE_CHECKPOINT_IDLE + && (force || core_should_take_checkpoint(spl)); + platform_mutex_unlock(&spl->checkpoint_state_lock); + if (!begin) { + return 0; + } + + log_handle *next = shard_log_create( + spl->cc, (shard_log_config *)spl->cfg.log_cfg, spl->heap_id); + if (next == NULL) { + platform_error_log( + "core_checkpoint_begin: shard_log_create failed; skipping\n"); + return 0; + } + log_head next_head = log_get_head(next); + + uint64 ticket = 0; + platform_mutex_lock(&spl->checkpoint_state_lock); + if (spl->checkpoint.phase == CORE_CHECKPOINT_IDLE) { + spl->checkpoint.pending_log = next; + spl->checkpoint.live_head = next_head; + spl->checkpoint.phase = CORE_CHECKPOINT_PENDING; + next = NULL; // handed off to the checkpoint + // Ours is the next completion to be counted. + ticket = spl->checkpoint.completions + 1; + } + platform_mutex_unlock(&spl->checkpoint_state_lock); + + if (next != NULL) { + // Lost a race with a concurrent rotation; discard the speculative log. + log_seal(next); + log_dec_ref(spl->cc, &next_head); + } + return ticket; +} + +/* The automatic, policy-driven arm, run after every rotation. */ +static void +core_checkpoint_maybe_begin(core_handle *spl) +{ + core_checkpoint_begin(spl, FALSE /* force */); +} + +/* + * Act on the size policy from the insert path. Called by core_insert() once + * the insert lock is released. + * + * A rotation is the only point at which the log can be cut, and a workload that + * overwrites in place updates the memtable without growing it -- so it may + * never fill a memtable, never rotate, and never give + * core_checkpoint_maybe_begin() (which only runs after a rotation) a chance to + * arm anything. Left to itself the log would grow without bound. + * + * Arming and then forcing the rotation cuts the log in a single rotation, since + * the rotate hook finds the checkpoint already PENDING. Only the thread that + * actually armed goes on to force, so concurrent inserters do not pile on. + * + * Not forcing the arm below is what makes this safe against a stale flag. The + * flag is only a hint -- sampled on some earlier insert, and readable by + * several threads at once -- so a thread can arrive here long after the log it + * observed was already cut. Passing force = FALSE has core_checkpoint_begin() + * re-check the policy under the state lock against the *current* log, which + * declines in exactly those cases (the fresh log reports zero bytes, or a + * checkpoint is still in flight) and proceeds only when another cut is + * genuinely due. Forcing here would instead cut again on a log that no longer + * needs it. + */ +static void +core_maybe_cut_oversized_log(core_handle *spl) +{ + if (!spl->log_reached_threshold) { + return; + } + if (core_checkpoint_begin(spl, FALSE /* force */) != 0) { + memtable_force_rotation(&spl->mt_ctxt); + } +} + +/* + * Begin, step 2 (inside the rotation critical section, insert lock held + * exclusively): swap the pre-created live log in. Registered as + * mt_ctxt.rotate. Every log writer holds the insert lock shared across its + * log_write, so once this store retires no writer is mid-write to, or will + * newly enter, the old log -- making the subsequent seal safe. + */ +static void +core_rotate_log(void *arg, uint64 finalized_generation) +{ + core_handle *spl = arg; + + platform_mutex_lock(&spl->checkpoint_state_lock); + if (spl->checkpoint.phase == CORE_CHECKPOINT_PENDING) { + spl->checkpoint.log_to_seal = spl->log; + spl->checkpoint.sealed_head = log_get_head(spl->log); + spl->log = spl->checkpoint.pending_log; + spl->checkpoint.pending_log = NULL; + /* + * The retiring log received everything up to and including + * finalized_generation, so the new live log's coverage starts at the next + * one. The retiring log's own start generation needs no tracking here: + * the superblock already records it, and superblock_log_cut() carries it + * across into the sealed slot. + */ + spl->checkpoint.live_start_generation = finalized_generation + 1; + spl->checkpoint.cut_generation = finalized_generation; + spl->checkpoint.phase = CORE_CHECKPOINT_SEALING; + + /* + * The size hint described the log we just retired; the fresh one has had + * nothing appended. This is the only place the hint is cleared, and the + * insert lock is held exclusively here, so it cannot race the stores in + * core_log_insert(). A rotation that does not cut leaves the hint alone, + * which is correct: the same log is still live and still oversized. + */ + spl->log_reached_threshold = FALSE; + } + platform_mutex_unlock(&spl->checkpoint_state_lock); +} + +/* + * Begin, step 3 (just after the rotation critical section): seal the + * swapped-out log and publish the cut. The swap drained and excluded all + * writers, so sealing is safe. + * + * Publishing here is what makes the cut crash-safe. The rotation moved inserts + * to the new live log, but the superblock still names the old one, so until + * this runs a crash would lose everything written to the new log. Order + * matters: the sealed log's pages must be durable before the superblock names + * it as sealed, since recovery replays it as-is. + */ +static void +core_checkpoint_seal_cut(core_handle *spl) +{ + platform_mutex_lock(&spl->checkpoint_state_lock); + log_handle *to_seal = NULL; + superblock_log_head live = {0}; + if (spl->checkpoint.phase == CORE_CHECKPOINT_SEALING) { + to_seal = spl->checkpoint.log_to_seal; + spl->checkpoint.log_to_seal = NULL; + spl->checkpoint.phase = CORE_CHECKPOINT_INCORPORATING; + live = core_log_to_superblock_log_head( + spl->checkpoint.live_head, spl->checkpoint.live_start_generation); + } + platform_mutex_unlock(&spl->checkpoint_state_lock); + + if (to_seal == NULL) { + return; + } + log_seal(to_seal); + + /* + * Serialize against any other superblock publisher (the superblock context + * is not thread safe). On failure the checkpoint still proceeds: the + * completion publish will record the correct final state; only this + * crash-protection window is left uncovered. + */ + platform_status rc = platform_mutex_lock(&spl->superblock_lock); + if (!SUCCESS(rc)) { + platform_error_log("core_checkpoint_seal_cut: lock failed: %s\n", + platform_status_to_string(rc)); + return; + } + rc = cache_writeback_dirty(spl->cc); + if (SUCCESS(rc)) { + rc = cache_durable_barrier(spl->cc); + } + if (SUCCESS(rc)) { + // The image still names the retiring log as live, so the cut moves it + // into the sealed slot, carrying its recorded start generation with it. + superblock_log_cut(&spl->superblock, live); + rc = superblock_make_durable(&spl->superblock); + } + if (!SUCCESS(rc)) { + platform_error_log("core_checkpoint_seal_cut: failed to publish the log " + "cut: %s\n", + platform_status_to_string(rc)); + } + platform_status unlock_rc = platform_mutex_unlock(&spl->superblock_lock); + platform_assert_status_ok(unlock_rc); +} + +/* + * Complete (after an incorporation): if the sealed log's generations are all + * folded into the trunk root, advance the durable root with the sealed slot + * cleared (which makes the root and superblock durable) and free the sealed + * log's extents. Called from the single-threaded incorporation path. + */ +static platform_status +core_maybe_complete_checkpoint(core_handle *spl) +{ + platform_mutex_lock(&spl->checkpoint_state_lock); + // The sealed log's cut generation is fully incorporated once it falls below + // the first unincorporated generation. When nothing has been retired, + // memtable_generation_retired() is UINT64_MAX and this wraps to 0, so the + // comparison is false without a sentinel check. + uint64 first_unincorporated = memtable_generation_retired(&spl->mt_ctxt) + 1; + bool32 complete = spl->checkpoint.phase == CORE_CHECKPOINT_INCORPORATING + && first_unincorporated > spl->checkpoint.cut_generation; + log_head sealed = {0}; + if (complete) { + sealed = spl->checkpoint.sealed_head; + spl->checkpoint.phase = CORE_CHECKPOINT_COMPLETING; + } + platform_mutex_unlock(&spl->checkpoint_state_lock); + + if (!complete) { + return STATUS_OK; + } + + /* + * The live log is already recorded (core_checkpoint_seal_cut() published the + * cut), so this only advances the root -- which is what lets snapshot_tree + * drop the now-covered sealed log. + */ + platform_status rc = core_checkpoint_commit_current_root(spl); + if (SUCCESS(rc)) { + // The sealed log's entries are now durably in the root; free its extents. + log_dec_ref(spl->cc, &sealed); + } else { + platform_error_log("core_maybe_complete_checkpoint: publish failed: %s\n", + platform_status_to_string(rc)); + } + + platform_mutex_lock(&spl->checkpoint_state_lock); + if (SUCCESS(rc)) { + ZERO_CONTENTS(&spl->checkpoint.sealed_head); + ZERO_CONTENTS(&spl->checkpoint.live_head); + spl->checkpoint.cut_generation = 0; + /* + * Count the completion only here, after log_dec_ref() above: waiters take + * this as proof the retired log's space is back. + */ + spl->checkpoint.completions++; + spl->checkpoint.phase = CORE_CHECKPOINT_IDLE; + } else { + // Leave the sealed slot intact and retry on a later incorporation. + spl->checkpoint.phase = CORE_CHECKPOINT_INCORPORATING; } + platform_mutex_unlock(&spl->checkpoint_state_lock); - return super; + // Outside the lock: this is a per-thread counter, so it needs none. + if (SUCCESS(rc) && spl->cfg.use_stats) { + spl->stats[platform_get_tid()].checkpoints_completed++; + } + return rc; } +/* + * Release any resources of an in-flight checkpoint during a quiesced shutdown, + * before the unmount/destroy publish. No locking: the caller has quiesced all + * inserts and incorporations. A completed checkpoint + * (INCORPORATING/COMPLETING) is normally already reaped by the quiesce drain; + * the residual cases below are defensive. + */ static void -core_release_super_block(core_handle *spl, page_handle *super_page) +core_checkpoint_cleanup_for_shutdown(core_handle *spl) { - cache_unget(spl->cc, super_page); + core_checkpoint_state *cp = &spl->checkpoint; + switch (cp->phase) { + case CORE_CHECKPOINT_IDLE: + break; + case CORE_CHECKPOINT_PENDING: + // The next live log was pre-created but never installed; discard it. + log_seal(cp->pending_log); + log_dec_ref(spl->cc, &cp->live_head); + break; + case CORE_CHECKPOINT_INCORPORATING: + case CORE_CHECKPOINT_COMPLETING: + // The sealed log is fully incorporated after quiesce. The shutdown + // publish records sealed=none, so just reclaim its extents here + // (before the map is persisted, so the map reflects the free). + log_dec_ref(spl->cc, &cp->sealed_head); + break; + case CORE_CHECKPOINT_SEALING: + default: + platform_assert( + FALSE, "unexpected checkpoint phase %d at shutdown", cp->phase); + } + ZERO_CONTENTS(cp); // phase == CORE_CHECKPOINT_IDLE } /* @@ -418,6 +856,7 @@ core_begin_memtable_insert(core_handle *spl, uint64 *generation, memtable **mt) static platform_status core_log_insert(core_handle *spl, + uint64 memtable_generation, key tuple_key, message msg, const btree_insert_results *insert_results) @@ -436,8 +875,31 @@ core_log_insert(core_handle *spl, merge_accumulator_is_null(&insert_results->msg_blob) ? msg : merge_accumulator_to_message(&insert_results->msg_blob); - int log_rc = - log_write(spl->log, tuple_key, log_msg, insert_results->leaf_generation); + int log_rc = log_write(spl->log, + tuple_key, + log_msg, + memtable_generation, + insert_results->leaf_generation); + + /* + * Sample the size policy while we still hold the shared insert lock, which + * is what makes reading spl->log safe (the live log is only swapped from + * inside the rotation critical section, which holds that lock exclusively). + * core_insert() acts on this once the lock is released. + * + * Only ever set it here, never clear it: this runs on every logged insert on + * every thread, and writing a shared field that often would bounce its cache + * line between cores for no reason. Leaving the common case read-only keeps + * the line shared. core_rotate_log() clears the flag when it cuts the log, + * which it does under the insert lock held exclusively -- so the clear + * cannot race this store. + */ + if (spl->cfg.checkpoint_log_size_bytes != 0 + && log_get_size(spl->log) >= spl->cfg.checkpoint_log_size_bytes) + { + spl->log_reached_threshold = TRUE; + } + return log_rc == 0 ? STATUS_OK : (platform_status){.r = log_rc}; } @@ -506,10 +968,16 @@ core_memtable_compact(core_handle *spl, uint64 generation, const threadid tid) } core_memtable_iterator_deinit(&btree_itor); + /* + * A forced rotation (see memtable_force_rotation(), used by + * core_checkpoint() when nothing rotates on its own) can finalize a memtable + * that received no inserts. btree_pack() already defines this case: an + * empty input yields num_tuples == 0 and root_addr == 0, allocating no page. + * The generation still retires; core_memtable_incorporate() recognizes the + * missing branch and skips the trunk incorporation. + */ new_branch->root_addr = req.root_addr; - platform_assert(req.num_tuples > 0); - btree_pack_req_deinit(&req, spl->heap_id); if (spl->cfg.use_stats) { uint64 comp_time = platform_timestamp_elapsed(comp_start); @@ -599,19 +1067,30 @@ core_memtable_incorporate(core_handle *spl, if (spl->cfg.use_stats) { flush_start = platform_get_timestamp(); } - rc = trunk_incorporate_prepare(&spl->trunk_context, cmt->branch.root_addr); - if (!SUCCESS(rc)) { - platform_error_log("trunk_incorporate_prepare failed: %s\n", - platform_status_to_string(rc)); - core_close_log_stream_if_enabled(spl, &stream); - core_memtable_mark_incorporation_failed(spl, generation, rc); - return rc; - } - btree_dec_ref( - spl->cc, spl->cfg.btree_cfg, cmt->branch.root_addr, PAGE_TYPE_BRANCH); - if (spl->cfg.use_stats) { - spl->stats[tid].memtable_flush_wait_time_ns += - platform_timestamp_elapsed(cmt->wait_start); + /* + * A forced rotation can retire a generation that received no inserts, in + * which case core_memtable_compact() produced no branch (root_addr == 0). + * There is nothing to fold into the trunk, and the trunk_incorporate_*() + * calls require a real branch (trunk_incorporate_prepare() asserts + * branch_addr != 0), so skip them; the generation still retires below. + */ + bool32 has_branch = (cmt->branch.root_addr != 0); + if (has_branch) { + rc = + trunk_incorporate_prepare(&spl->trunk_context, cmt->branch.root_addr); + if (!SUCCESS(rc)) { + platform_error_log("trunk_incorporate_prepare failed: %s\n", + platform_status_to_string(rc)); + core_close_log_stream_if_enabled(spl, &stream); + core_memtable_mark_incorporation_failed(spl, generation, rc); + return rc; + } + btree_dec_ref( + spl->cc, spl->cfg.btree_cfg, cmt->branch.root_addr, PAGE_TYPE_BRANCH); + if (spl->cfg.use_stats) { + spl->stats[tid].memtable_flush_wait_time_ns += + platform_timestamp_elapsed(cmt->wait_start); + } } core_log_stream_if_enabled( @@ -635,10 +1114,14 @@ core_memtable_incorporate(core_handle *spl, memtable_transition( mt, MEMTABLE_STATE_INCORPORATING, MEMTABLE_STATE_INCORPORATED); memtable_increment_to_generation_retired(&spl->mt_ctxt, generation); - trunk_incorporate_commit(&spl->trunk_context); + if (has_branch) { + trunk_incorporate_commit(&spl->trunk_context); + } memtable_unblock_lookups(&spl->mt_ctxt); - trunk_incorporate_cleanup(&spl->trunk_context); + if (has_branch) { + trunk_incorporate_cleanup(&spl->trunk_context); + } core_close_log_stream_if_enabled(spl, &stream); @@ -682,6 +1165,9 @@ core_memtable_flush_internal(core_handle *spl, uint64 generation) } generation++; } while (core_try_continue_incorporate(spl, generation)); + + // A checkpoint's sealed log may now be fully incorporated; complete it. + core_maybe_complete_checkpoint(spl); out: return STATUS_OK; } @@ -701,11 +1187,6 @@ core_memtable_flush_internal_virtual(task *arg) /* * Function to trigger a memtable incorporation. Called in the context of * the foreground doing insertions. - * If background threads are not enabled, this function does the entire memtable - * incorporation inline. - * If background threads are enabled, this function just queues up the task to - * carry out the incorporation, swaps the curr_memtable pointer, claims the - * root and returns. */ static void core_memtable_flush(core_handle *spl, uint64 generation) @@ -724,7 +1205,16 @@ static void core_memtable_flush_virtual(void *arg, uint64 generation) { core_handle *spl = arg; + + // Begin, step 3: if this rotation's in-CS hook swapped the live log, seal + // the old one now that the critical section has been released. + core_checkpoint_seal_cut(spl); + core_memtable_flush(spl, generation); + + // Begin, step 1: decide whether the next rotation should start a checkpoint, + // pre-creating its live log outside any critical section. + core_checkpoint_maybe_begin(spl); } static inline uint64 @@ -776,6 +1266,12 @@ core_memtable_lookup(core_handle *spl, bool32 memtable_is_compacted; uint64 root_addr = core_memtable_root_addr_for_lookup( spl, generation, &memtable_is_compacted, NULL); + if (memtable_is_compacted && root_addr == 0) { + // A forced rotation can retire an empty generation, whose compacted + // branch has no root page. It holds no tuples, so there is nothing to + // search -- equivalent to finding nothing here. + return STATUS_OK; + } page_type type = memtable_is_compacted ? PAGE_TYPE_BRANCH : PAGE_TYPE_MEMTABLE; @@ -1092,13 +1588,21 @@ core_range_iterator_init(core_handle *spl, bool32 active; uint64 root_addr = core_memtable_root_addr_for_lookup(spl, mt_gen, &compacted, &active); - range_itor->compacted[range_itor->num_branches] = compacted; // Only READY memtables can be modified while this iterator is live. + // Determined before the empty-generation skip below so that skipping + // cannot affect which generation is treated as the first one. if (range_itor->num_branches == 0) { first_memtable_copy_nodes = active; } else { debug_assert(!active); } + if (compacted && root_addr == 0) { + // A forced rotation can retire an empty generation, whose compacted + // branch has no root page. It contributes no tuples, so there is + // nothing to merge from it. + continue; + } + range_itor->compacted[range_itor->num_branches] = compacted; if (compacted) { btree_inc_ref(spl->cc, spl->cfg.btree_cfg, root_addr); } else { @@ -1554,7 +2058,7 @@ core_insert(core_handle *spl, goto end_insert; } - rc = core_log_insert(spl, tuple_key, data, &insert_results); + rc = core_log_insert(spl, generation, tuple_key, data, &insert_results); if (!SUCCESS(rc)) { goto end_insert; } @@ -1581,6 +2085,8 @@ core_insert(core_handle *spl, btree_insert_results_deinit(&insert_results); task_perform_one_if_needed(spl->ts, spl->cfg.queue_scale_percent); + // The insert lock is released by here, so this may force a rotation. + core_maybe_cut_oversized_log(spl); if (spl->cfg.use_stats) { switch (message_class(data)) { @@ -1865,6 +2371,7 @@ core_mkfs(core_handle *spl, core_config *cfg, allocator *al, cache *cc, + io_handle *io, task_system *ts, allocator_root_id id, platform_heap_id hid) @@ -1879,32 +2386,61 @@ core_mkfs(core_handle *spl, spl->heap_id = hid; spl->ts = ts; + platform_status rc = core_locks_init(spl); + if (!SUCCESS(rc)) { + platform_error_log("core_mkfs: lock initialization failed: %s\n", + platform_status_to_string(rc)); + return rc; + } + + // Fresh superblock: geometry, empty tree table, allocation state invalid. + allocator_config *allocator_cfg = allocator_get_config(al); + rc = superblock_context_init(&spl->superblock, io, allocator_cfg, hid); + if (!SUCCESS(rc)) { + platform_error_log("core_mkfs: superblock_context_init failed: %s\n", + platform_status_to_string(rc)); + goto deinit_locks; + } + rc = superblock_format(&spl->superblock, allocator_cfg); + if (!SUCCESS(rc)) { + platform_error_log("core_mkfs: superblock_format failed: %s\n", + platform_status_to_string(rc)); + goto deinit_superblock; + } + // set up the memtable context memtable_config *mt_cfg = &spl->cfg.mt_cfg; - platform_status rc = memtable_context_init(&spl->mt_ctxt, - spl->heap_id, - cc, - mt_cfg, - core_memtable_flush_virtual, - spl); + rc = memtable_context_init(&spl->mt_ctxt, + spl->heap_id, + cc, + mt_cfg, + core_rotate_log, + core_memtable_flush_virtual, + spl); if (!SUCCESS(rc)) { platform_error_log("core_mkfs: memtable_context_init failed: %s\n", platform_status_to_string(rc)); - return rc; + goto deinit_superblock; } // set up the log if (spl->cfg.use_log) { - spl->log = log_create(cc, spl->cfg.log_cfg, spl->heap_id); + spl->log = shard_log_create( + cc, (shard_log_config *)spl->cfg.log_cfg, spl->heap_id); if (spl->log == NULL) { - platform_error_log("core_mkfs: log_create failed\n"); + platform_error_log("core_mkfs: shard_log_create failed\n"); rc = STATUS_NO_MEMORY; goto deinit_memtable_context; } } - rc = trunk_context_init( - &spl->trunk_context, spl->cfg.trunk_node_cfg, hid, cc, al, ts, 0); + rc = trunk_context_init(&spl->trunk_context, + spl->cfg.trunk_node_cfg, + hid, + cc, + al, + ts, + (trunk_snapshot){.root_addr = 0}); if (!SUCCESS(rc)) { platform_error_log("core_mkfs: trunk_context_init failed: %s\n", platform_status_to_string(rc)); @@ -1918,10 +2454,22 @@ core_mkfs(core_handle *spl, goto deinit_trunk_context; } - rc = core_set_super_block(spl, FALSE, FALSE, TRUE); + /* + * Establish the initial (empty) tree record and install this instance's log, + * then publish both durably in one write. The formatted image has no live + * log, so the cut leaves the sealed slot empty; the log covers generations + * from 0. + */ + if (spl->cfg.use_log) { + superblock_log_cut( + &spl->superblock, + core_log_to_superblock_log_head(log_get_head(spl->log), 0)); + } + rc = core_checkpoint_commit_current_root(spl); if (!SUCCESS(rc)) { - platform_error_log("core_mkfs: core_set_super_block failed: %s\n", - platform_status_to_string(rc)); + platform_error_log( + "core_mkfs: core_checkpoint_commit_current_root failed: %s\n", + platform_status_to_string(rc)); goto deinit_stats; } return STATUS_OK; @@ -1937,6 +2485,10 @@ core_mkfs(core_handle *spl, } deinit_memtable_context: memtable_context_deinit(&spl->mt_ctxt); +deinit_superblock: + superblock_context_deinit(&spl->superblock); +deinit_locks: + core_locks_deinit(spl); return rc; } @@ -1948,6 +2500,7 @@ core_mount(core_handle *spl, core_config *cfg, allocator *al, cache *cc, + io_handle *io, task_system *ts, allocator_root_id id, platform_heap_id hid) @@ -1962,43 +2515,106 @@ core_mount(core_handle *spl, spl->heap_id = hid; spl->ts = ts; - // find the unmounted super block - uint64 root_addr = 0; - uint64 latest_timestamp = 0; - page_handle *super_page; - core_super_block *super = core_get_super_block_if_valid(spl, &super_page); - if (super != NULL) { - if (super->unmounted && super->timestamp > latest_timestamp) { - root_addr = super->root_addr; - latest_timestamp = super->timestamp; - } - core_release_super_block(spl, super_page); + platform_status rc = core_locks_init(spl); + if (!SUCCESS(rc)) { + platform_error_log("core_mount: lock initialization failed: %s\n", + platform_status_to_string(rc)); + return rc; } + // Read the superblock (newest valid A/B copy; validates geometry). + allocator_config *allocator_cfg = allocator_get_config(al); + rc = superblock_context_init(&spl->superblock, io, allocator_cfg, hid); + if (!SUCCESS(rc)) { + platform_error_log("core_mount: superblock_context_init failed: %s\n", + platform_status_to_string(rc)); + goto deinit_locks; + } + rc = superblock_mount(&spl->superblock, allocator_cfg); + if (!SUCCESS(rc)) { + platform_error_log("core_mount: superblock_mount failed: %s\n", + platform_status_to_string(rc)); + goto deinit_superblock; + } + + superblock_tree_record rec; + superblock_get_tree_record(&spl->superblock, &rec); + + /* + * Preserve the historical clean-only mount rule for this first format + * slice: crash recovery (log replay + allocator rebuild) is not wired yet, + * so only a clean at-rest instance -- one whose allocation state is still + * valid -- may supply the root. A valid allocation state is published only + * at the end of a clean unmount, so it is the single at-rest signal (no + * separate per-tree clean flag is needed; see superblock.h). + */ + bool32 rebuild = !superblock_allocation_state_valid(&spl->superblock); + if (rebuild) { + platform_error_log("core_mount: root id %lu requires crash recovery\n", + spl->id); + rc = STATUS_INVALID_STATE; + goto deinit_superblock; + } + + /* + * Load the trusted refcount map (rebuild == FALSE on this path). This must + * precede trunk_snapshot_create_from_addr(), which increments the root's + * refcount in the now-loaded map. + */ + rc = allocator_load_refcounts(al); + if (!SUCCESS(rc)) { + platform_error_log("core_mount: allocator_load_refcounts failed: %s\n", + platform_status_to_string(rc)); + goto deinit_superblock; + } + + uint64 root_addr = rec.root_addr; + // The record already stores the first unincorporated generation, which is + // exactly where the memtable resumes (0 for a fresh, never-incorporated db). + uint64 resume_generation = rec.first_unincorporated_generation; + memtable_config *mt_cfg = &spl->cfg.mt_cfg; - platform_status rc = memtable_context_init(&spl->mt_ctxt, - spl->heap_id, - cc, - mt_cfg, - core_memtable_flush_virtual, - spl); + rc = memtable_context_init_at_generation(&spl->mt_ctxt, + spl->heap_id, + cc, + mt_cfg, + core_rotate_log, + core_memtable_flush_virtual, + spl, + resume_generation); if (!SUCCESS(rc)) { - platform_error_log("core_mount: memtable_context_init failed: %s\n", + platform_error_log("core_mount: memtable_context_init_at_generation " + "failed: %s\n", platform_status_to_string(rc)); - return rc; + goto deinit_superblock; } if (spl->cfg.use_log) { - spl->log = log_create(cc, spl->cfg.log_cfg, spl->heap_id); + spl->log = shard_log_create( + cc, (shard_log_config *)spl->cfg.log_cfg, spl->heap_id); if (spl->log == NULL) { - platform_error_log("core_mount: log_create failed\n"); + platform_error_log("core_mount: shard_log_create failed\n"); rc = STATUS_NO_MEMORY; goto deinit_memtable_context; } } - rc = trunk_context_init( - &spl->trunk_context, spl->cfg.trunk_node_cfg, hid, cc, al, ts, root_addr); + trunk_snapshot root_snapshot; + rc = trunk_snapshot_create_from_addr(al, root_addr, &root_snapshot); + if (!SUCCESS(rc)) { + platform_error_log( + "core_mount: trunk_snapshot_create_from_addr failed: %s\n", + platform_status_to_string(rc)); + goto deinit_log; + } + + rc = trunk_context_init(&spl->trunk_context, + spl->cfg.trunk_node_cfg, + hid, + cc, + al, + ts, + root_snapshot); if (!SUCCESS(rc)) { platform_error_log("core_mount: trunk_context_init failed: %s\n", platform_status_to_string(rc)); @@ -2012,9 +2628,23 @@ core_mount(core_handle *spl, goto deinit_trunk_context; } - rc = core_set_super_block(spl, FALSE, FALSE, FALSE); + /* + * Mark dirty: cut this session's fresh live log and invalidate the persisted + * allocation state, before any allocation diverges the in-memory map from + * disk. A crash after this forces the next mount into recovery instead of + * silently reverting to this now-stale root. The root is unchanged; a clean + * mount has no prior live log, so the sealed slot stays empty. This + * session's log receives generations from the resume generation onward. + */ + superblock_log_cut(&spl->superblock, + spl->cfg.use_log + ? core_log_to_superblock_log_head( + log_get_head(spl->log), resume_generation) + : (superblock_log_head){0}); + rc = superblock_make_durable(&spl->superblock); if (!SUCCESS(rc)) { - platform_error_log("core_mount: core_set_super_block failed: %s\n", + platform_error_log("core_mount: mark-dirty superblock_make_durable " + "failed: %s\n", platform_status_to_string(rc)); goto deinit_stats; } @@ -2031,6 +2661,10 @@ core_mount(core_handle *spl, } deinit_memtable_context: memtable_context_deinit(&spl->mt_ctxt); +deinit_superblock: + superblock_context_deinit(&spl->superblock); +deinit_locks: + core_locks_deinit(spl); return rc; } @@ -2082,21 +2716,23 @@ core_report_unincorporated_memtables(core_handle *spl) /* * This function is only safe to call when all other calls to spl have returned. + * It intentionally leaves the memtable and log contexts live: the clean + * checkpoint record needs both after final incorporation has quiesced. */ -void -core_prepare_for_shutdown(core_handle *spl) +static void +core_quiesce_for_shutdown(core_handle *spl) { // write current memtable to disk // (any others must already be flushing/flushed) if (!memtable_is_empty(&spl->mt_ctxt)) { /* - * memtable_force_finalize is not thread safe. Note also, we do not hold - * the insert lock or rotate while flushing the memtable. + * memtable_force_rotation is not thread safe. It dispatches the flush + * itself (via the process callback), which also resolves any checkpoint + * log cut its rotate hook just made. That callback may arm a fresh + * checkpoint; core_checkpoint_cleanup_for_shutdown() discards it. */ - - uint64 generation = memtable_force_finalize(&spl->mt_ctxt); - core_memtable_flush(spl, generation); + memtable_force_rotation(&spl->mt_ctxt); } // finish any outstanding tasks and destroy task system for this table. @@ -2104,17 +2740,116 @@ core_prepare_for_shutdown(core_handle *spl) platform_assert_status_ok(rc); core_report_unincorporated_memtables(spl); +} - // destroy memtable context (and its memtables) - memtable_context_deinit(&spl->mt_ctxt); +/* + * Seal the live log at shutdown -- finalizes its pages and frees the handle -- + * and return its identity so the caller can free the extents with log_dec_ref() + * once the cache is flushed. A clean shutdown has folded everything into the + * durable root, so the live log is fully incorporated and discardable. Returns + * an empty descriptor when logging is disabled. + */ +static log_head +core_seal_live_log(core_handle *spl) +{ + if (!spl->cfg.use_log || spl->log == NULL) { + return (log_head){0}; + } + log_head info = log_get_head(spl->log); + log_seal(spl->log); + spl->log = NULL; + return info; +} - // release the log - if (spl->cfg.use_log) { - platform_free(spl->heap_id, spl->log); +/* + * Take a checkpoint: make every modification that completed before this call + * durable in the trunk root, reclaiming the retired log's space, and block + * until that is done. Safe to call on a running system with concurrent + * inserts. + * + * Durability reduces to a generation bound. Inserts land in the currently + * active memtable generation and generations only advance, so everything + * already inserted is in a generation at or below the one active at entry -- + * the target. Once the target is incorporated, all of it is in the trunk's COW + * root, and committing that root makes it durable. + * + * Reclaiming log space needs a log cut, which happens only when a rotation + * finds a checkpoint armed. So we arm one up front and then see it through: + * its completion frees the retired log's extents. This is what lets an + * application turn the interval policy off and manage log space entirely + * through this call. Arming is best effort -- only one checkpoint can be in + * flight, so if one already is we ride it out and leave reclamation to it. + * Durability does not depend on any of this. + * + * Both halves wait on a rotation: the target cannot be incorporated until + * something finalizes it, and the cut cannot happen without one either. Insert + * traffic normally provides it; if none arrives within rotation_timeout_ns we + * force one. Forcing is safe even with an empty memtable -- that generation + * retires with no branch at all (see core_memtable_compact()) -- but note that + * a caller polling a quiescent database will force a rotation per call, + * spending a generation and a log extent each time. A longer timeout lets real + * traffic drive the cut instead. + */ +platform_status +core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns) +{ + uint64 target = memtable_generation(&spl->mt_ctxt); + + uint64 ticket = core_checkpoint_begin(spl, TRUE /* force */); + + uint64 wait = 100; + timestamp deadline = platform_get_timestamp(); + while (TRUE) { + bool32 incorporated = + memtable_generation_retired(&spl->mt_ctxt) + 1 > target; + core_checkpoint_status status = core_checkpoint_status_get(spl); + /* + * Done once the target is durable-able and, if we started a checkpoint, + * that specific checkpoint has completed -- which is what freed its + * retired log. Comparing against our own ticket rather than "nothing in + * flight" means neither an unrelated checkpoint nor one armed after ours + * can hold us up. + */ + bool32 ours_completed = (ticket == 0) || (status.completions >= ticket); + if (incorporated && ours_completed) { + break; + } + + /* + * Force a rotation if either half is still waiting on one: the target is + * still the active generation, or our checkpoint has yet to cut. Once + * neither holds, the remaining work is just draining flushes. + * + * The second clause is not redundant. Another thread can finalize the + * target in the window between reading target and arming, which leaves + * the first clause false while our checkpoint still needs a rotation to + * cut. On an otherwise idle system nothing would ever provide one, and + * the wait for our completion would never finish. + */ + bool32 needs_rotation = + memtable_generation(&spl->mt_ctxt) == target + || (ticket != 0 && status.phase == CORE_CHECKPOINT_PENDING); + if (needs_rotation + && rotation_timeout_ns <= platform_timestamp_elapsed(deadline)) + { + memtable_force_rotation(&spl->mt_ctxt); // dispatches the flush itself + deadline = platform_get_timestamp(); + } + + task_perform_one_if_needed(spl->ts, 0); + platform_sleep_ns(wait); + wait = wait > 2048 ? wait : 2 * wait; } - // flush all dirty pages in the cache - cache_flush(spl->cc); + /* + * Advance the durable root. The log slots are left exactly as the cut + * protocol published them, so this cannot orphan a log; snapshot_tree only + * drops the sealed log if this root already covers it. A completed + * checkpoint has already committed an equivalent root -- republishing is + * harmless -- and this is still required when nothing was armed (or logging + * is off). + */ + return core_checkpoint_commit_current_root(spl); } /* @@ -2126,14 +2861,77 @@ core_unmount(core_handle *spl) { platform_status rc; - core_prepare_for_shutdown(spl); - rc = core_set_super_block(spl, FALSE, TRUE, FALSE); + /* + * Quiescing leaves the memtable and log contexts live so publication can + * atomically capture the retired generation and root. Teardown is safe + * regardless of publication success. + */ + core_quiesce_for_shutdown(spl); + + // Reclaim any in-flight checkpoint's logs before the unmount publish. + core_checkpoint_cleanup_for_shutdown(spl); + + /* + * The clean-unmount root incorporates everything, so the live log is fully + * folded and discardable. Seal it (frees the handle) now; free its extents + * after the cache flush below. + */ + log_head live_log = core_seal_live_log(spl); + + /* + * Part A: publish the clean-unmount root with both log slots cleared (no + * live or sealed log at rest -- their extents are freed just below, so no + * reference to them may survive). Both transitions go out in the single + * publish the commit performs. Allocation state stays invalid here; it + * becomes valid only in Part B, after the map is persisted. + */ + superblock_discard_logs(&spl->superblock); + rc = core_checkpoint_commit_current_root(spl); if (!SUCCESS(rc)) { - platform_error_log("core_unmount: failed to update super block: %s\n", + platform_error_log("core_unmount: failed to publish unmount root: %s\n", platform_status_to_string(rc)); } + + // Keep this after publication above: it supplies the generation cut. + memtable_context_deinit(&spl->mt_ctxt); + + // Flush all dirty pages. The live log has already been sealed (above); free + // its extents now that the cache is flushed, before the map is persisted + // (Part B) so the persisted map reflects the free. + cache_flush(spl->cc); + log_dec_ref(spl->cc, &live_log); + /* + * Release the context's live root reference before persisting the map, so + * the persisted refcounts reflect exactly the durable record's single + * reference to the root. + */ trunk_context_deinit(&spl->trunk_context); + + /* + * Part B: only after a clean Part A publish, persist the refcount map and + * republish a valid allocation state pointing at it. The map is made + * durable before the "map is trustworthy" flag, and that flag becomes + * durable only after the root it must agree with (Part A). On a Part A + * failure we leave the allocation state invalid so the next open rebuilds. + */ + if (SUCCESS(rc)) { + uint64 map_addr; + platform_status prc = allocator_persist(spl->al, &map_addr); + if (SUCCESS(prc)) { + superblock_snapshot_allocator(&spl->superblock, map_addr); + prc = superblock_make_durable(&spl->superblock); + } + if (!SUCCESS(prc)) { + platform_error_log("core_unmount: failed to publish clean allocation " + "state: %s\n", + platform_status_to_string(prc)); + rc = prc; + } + } + + superblock_context_deinit(&spl->superblock); core_destroy_stats(spl); + core_locks_deinit(spl); return rc; } @@ -2143,12 +2941,49 @@ core_unmount(core_handle *spl) void core_destroy(core_handle *spl) { - core_prepare_for_shutdown(spl); + core_quiesce_for_shutdown(spl); + + // Reclaim any in-flight checkpoint's logs before teardown. + core_checkpoint_cleanup_for_shutdown(spl); + + /* + * Release the reference the published tree record holds on its root before + * tearing down the context (which releases the context's own live root + * reference). Together these free the whole tree. Release must precede + * trunk_context_deinit(): trunk_snapshot_release() needs the live context. + */ + superblock_tree_record rec; + superblock_get_tree_record(&spl->superblock, &rec); + if (rec.root_addr != 0) { + trunk_snapshot old_snapshot = {.root_addr = rec.root_addr}; + platform_status rc = + trunk_snapshot_release(&spl->trunk_context, &old_snapshot); + if (!SUCCESS(rc)) { + platform_error_log( + "core_destroy: failed to release root addr %lu: %s\n", + rec.root_addr, + platform_status_to_string(rc)); + } + } + + // Discard the live log too: seal (frees the handle), then free its extents + // after the cache flush. + log_head live_log = core_seal_live_log(spl); + memtable_context_deinit(&spl->mt_ctxt); + cache_flush(spl->cc); + log_dec_ref(spl->cc, &live_log); trunk_context_deinit(&spl->trunk_context); - // clear out this splinter table from the meta page. - allocator_remove_super_addr(spl->al, spl->id); + /* + * A destroyed instance must not be reopened. The mount-time mark-dirty + * already left the on-disk allocation state invalid, so we deliberately do + * not persist the map or publish a valid state here: the freed root and its + * subtree stay unreachable, and the next mount rejects the device + * (allocation state invalid) rather than trusting a now-freed root. + */ + superblock_context_deinit(&spl->superblock); core_destroy_stats(spl); + core_locks_deinit(spl); } @@ -2181,27 +3016,34 @@ core_print_space_use(platform_log_handle *log_handle, core_handle *spl) /* * core_print_super_block() * - * Fetch a super-block for a running Splinter instance, and print its - * contents. + * Print this instance's superblock tree record and the persisted allocation + * state. */ void core_print_super_block(platform_log_handle *log_handle, core_handle *spl) { - page_handle *super_page; - core_super_block *super = core_get_super_block_if_valid(spl, &super_page); - if (super == NULL) { - return; - } + superblock_tree_record rec; + superblock_get_tree_record(&spl->superblock, &rec); - platform_log(log_handle, "Superblock root_addr=%lu {\n", super->root_addr); - platform_log(log_handle, "log_meta_addr=%lu\n", super->log_meta_addr); platform_log(log_handle, - "timestamp=%lu, checkpointed=%d, unmounted=%d\n", - super->timestamp, - super->checkpointed, - super->unmounted); - platform_log(log_handle, "}\n\n"); - core_release_super_block(spl, super_page); + "Superblock tree record root_id=%lu {\n" + " root_addr=%lu first_unincorporated_generation=%lu\n" + " live_log: meta_addr=%lu addr=%lu magic=%lu\n" + " sealed_log: meta_addr=%lu addr=%lu magic=%lu\n" + " allocation_state: %s (addr=%lu)\n" + "}\n\n", + spl->id, + rec.root_addr, + rec.first_unincorporated_generation, + rec.live_log.meta_addr, + rec.live_log.addr, + rec.live_log.magic, + rec.sealed_log.meta_addr, + rec.sealed_log.addr, + rec.sealed_log.magic, + superblock_allocation_state_valid(&spl->superblock) ? "valid" + : "invalid", + superblock_allocation_state_addr(&spl->superblock)); } // clang-format off @@ -2288,6 +3130,7 @@ core_print_insertion_stats(platform_log_handle *log_handle, const core_handle *s spl->stats[thr_i].memtable_flush_time_max_ns; } global->memtable_flush_root_full += spl->stats[thr_i].memtable_flush_root_full; + global->checkpoints_completed += spl->stats[thr_i].checkpoints_completed; } platform_log(log_handle, "Overall Statistics\n"); @@ -2299,6 +3142,8 @@ core_print_insertion_stats(platform_log_handle *log_handle, const core_handle *s platform_log(log_handle, "------------------------------------------------------------------------------------\n"); platform_log(log_handle, "| root stalls: %10lu\n", global->memtable_flush_root_full); platform_log(log_handle, "------------------------------------------------------------------------------------\n"); + platform_log(log_handle, "| checkpoints: %10lu\n", global->checkpoints_completed); + platform_log(log_handle, "------------------------------------------------------------------------------------\n"); platform_log(log_handle, "\n"); platform_log(log_handle, "Latency Histogram Statistics\n"); @@ -2405,6 +3250,9 @@ core_print_lookup(core_handle *spl, key target, platform_log_handle *log_handle) bool32 memtable_is_compacted; uint64 root_addr = core_memtable_root_addr_for_lookup( spl, mt_gen, &memtable_is_compacted, NULL); + if (memtable_is_compacted && root_addr == 0) { + continue; // empty generation: no branch to look up in + } platform_status rc; rc = btree_lookup(spl->cc, @@ -2483,6 +3331,7 @@ core_config_init(core_config *core_cfg, uint64 queue_scale_percent, uint64 prefetch_budget, bool32 use_log, + uint64 checkpoint_log_size_bytes, bool32 use_stats, bool32 verbose_logging, platform_log_handle *log_handle) @@ -2497,12 +3346,13 @@ core_config_init(core_config *core_cfg, core_cfg->trunk_node_cfg = trunk_node_cfg; core_cfg->log_cfg = log_cfg; - core_cfg->queue_scale_percent = queue_scale_percent; - core_cfg->prefetch_budget = prefetch_budget; - core_cfg->use_log = use_log; - core_cfg->use_stats = use_stats; - core_cfg->verbose_logging_enabled = verbose_logging; - core_cfg->log_handle = log_handle; + core_cfg->queue_scale_percent = queue_scale_percent; + core_cfg->prefetch_budget = prefetch_budget; + core_cfg->use_log = use_log; + core_cfg->checkpoint_log_size_bytes = checkpoint_log_size_bytes; + core_cfg->use_stats = use_stats; + core_cfg->verbose_logging_enabled = verbose_logging; + core_cfg->log_handle = log_handle; memtable_config_init(&core_cfg->mt_cfg, core_cfg->btree_cfg, diff --git a/src/core.h b/src/core.h index 94c10f97..f4264891 100644 --- a/src/core.h +++ b/src/core.h @@ -14,6 +14,7 @@ #include "log.h" #include "trunk.h" #include "histogram.h" +#include "superblock.h" /* * Upper-bound on most number of branches that we can find our lookup-key in. @@ -50,7 +51,21 @@ typedef struct core_config { data_config *data_cfg; bool32 use_log; log_config *log_cfg; - trunk_config *trunk_node_cfg; + /* + * Automatic-checkpoint policy: take a checkpoint (rotate the log and advance + * the durable root) once the live log reaches this many bytes. 0 disables + * automatic checkpoints, leaving durability and log reclamation entirely to + * explicit core_checkpoint() calls. Defaults to the cache size. + * + * Sizing the trigger by log bytes rather than by memtable generations + * matters because the two are independent: a workload that repeatedly + * overwrites the same keys updates the memtable in place, so it may never + * fill a memtable or advance a generation, while every write still appends + * to the log. A generation-based trigger would never fire and the log would + * grow without bound. + */ + uint64 checkpoint_log_size_bytes; + trunk_config *trunk_node_cfg; // verbose logging bool32 verbose_logging_enabled; @@ -82,6 +97,15 @@ typedef struct core_stats { uint64 discarded_deletes; + /* + * Checkpoints that ran to completion -- meaning the durable root advanced + * and the retired log's space came back. Counted here for reporting only; + * the authoritative count the checkpoint machinery waits on is + * core_checkpoint_state.completions, which must stay monotonic and so is not + * affected by core_reset_stats(). + */ + uint64 checkpoints_completed; + uint64 lookups_found; uint64 lookups_not_found; } PLATFORM_CACHELINE_ALIGNED core_stats; @@ -93,6 +117,58 @@ typedef struct core_branch { typedef struct core_handle core_handle; +/* + * Incorporation-driven checkpoint (two-log protocol) state machine. + * + * IDLE no checkpoint in progress. + * PENDING the next live log is pre-created; the next memtable rotation + * will swap it in under the insert lock. + * SEALING the rotation swapped the new live log in; the old log still + * needs sealing (which will be performed just after the + * rotation critical section). + * INCORPORATING the old log is sealed; waiting for its generations to + * be incorporated into the trunk root. + * COMPLETING the completion publish (advance root, clear sealed slot) is + * in flight. + * + * The only transition that touches the shared spl->log pointer (PENDING -> + * SEALING) runs inside the memtable rotation critical section, where the insert + * lock is held exclusively; every log writer holds that lock shared across its + * log_write, so no writer can be mid-write to, or newly enter, the old log once + * it is swapped out. All other fields are guarded by checkpoint_state_lock, + * which is only ever held for brief, I/O-free updates. + */ +typedef enum core_checkpoint_phase { + CORE_CHECKPOINT_IDLE = 0, + CORE_CHECKPOINT_PENDING, + CORE_CHECKPOINT_SEALING, + CORE_CHECKPOINT_INCORPORATING, + CORE_CHECKPOINT_COMPLETING, +} core_checkpoint_phase; + +typedef struct core_checkpoint_state { + core_checkpoint_phase phase; + log_handle *pending_log; // next live log, pre-created (PENDING) + log_handle *log_to_seal; // old live log awaiting seal (SEALING) + log_head sealed_head; // identity of the sealed log (reclaim) + log_head live_head; // identity of the new live log + // First generation the new live log receives, recorded in the superblock as + // its coverage start. The retiring log's start needs no tracking: the + // superblock already holds it and carries it into the sealed slot. + uint64 live_start_generation; + uint64 cut_generation; // complete once retired >= this + /* + * Checkpoints completed so far. Bumped only after the completion has freed + * the retired log, so it is the one observable meaning "that checkpoint's + * space is back" -- every superblock-visible signal is necessarily written + * before the free, since the superblock must stop naming a log before its + * extents are released. core_checkpoint_begin() hands out `completions + 1` + * as a ticket so a caller can wait for its own checkpoint rather than merely + * for "none in flight." + */ + uint64 completions; +} core_checkpoint_state; + typedef struct core_memtable_args { core_handle *spl; uint64 generation; @@ -109,7 +185,6 @@ struct core_handle { core_config cfg; platform_heap_id heap_id; - uint64 super_block_idx; allocator_root_id id; allocator *al; @@ -119,6 +194,43 @@ struct core_handle { trunk_context trunk_context; memtable_context mt_ctxt; + /* + * Durable instance metadata. core owns the in-memory superblock context + * (allocated at mkfs/mount, torn down at unmount/destroy): it borrows the + * geometry, reads its tree record, and publishes root advances plus + * allocation-state transitions. For now the instance holds a single tree; + * when multi-tree support lands this ownership hoists to an instance level + * that per-tree cores borrow. + */ + /* Serializes snapshot cuts and superblock publication. */ + platform_mutex superblock_lock; + superblock_context superblock; + + /* + * Incorporation-driven checkpoint state. checkpoint_state_lock guards the + * fields of `checkpoint` (and is held while `log` is swapped); it is only + * ever held for brief, I/O-free updates (never across a barrier), so taking + * it inside the memtable rotation critical section cannot stall inserts on + * I/O. + */ + platform_mutex checkpoint_state_lock; + core_checkpoint_state checkpoint; + + /* + * Hint that the live log has reached checkpoint_log_size_bytes. Set by + * core_log_insert() -- which already holds the shared insert lock, the same + * lock that excludes the log swap, so it can read `log` safely -- and acted + * on by core_insert() once that lock is released. Cleared only by + * core_rotate_log() when it cuts the log, under the insert lock held + * exclusively, so set and clear cannot race. + * + * Set-only on the insert path so the common case does no store and the cache + * line stays shared across cores. Being merely a hint, a stale TRUE costs + * nothing: core_maybe_cut_oversized_log() re-checks the policy against the + * current log before acting. + */ + bool32 log_reached_threshold; + core_stats *stats; core_compacted_memtable compacted_memtable[MAX_MEMTABLES]; @@ -223,6 +335,7 @@ core_mkfs(core_handle *spl, core_config *cfg, allocator *al, cache *cc, + io_handle *io, task_system *ts, allocator_root_id id, platform_heap_id hid); @@ -232,10 +345,28 @@ core_mount(core_handle *spl, core_config *cfg, allocator *al, cache *cc, + io_handle *io, task_system *ts, allocator_root_id id, platform_heap_id hid); +/* + * Take a checkpoint: make every modification made before this call durable in + * the trunk root, and reclaim the space of the log it retires. Blocks until + * both are done. Safe to call on a running system with concurrent inserts. + * + * Reclamation makes this sufficient on its own, so an application can set + * checkpoint_log_size_bytes to 0 (no automatic checkpoints) and manage + * durability and log space entirely through this call. + * + * Both halves need a memtable rotation, which insert traffic normally triggers. + * If none occurs within rotation_timeout_ns, the rotation is forced; pass 0 to + * force it immediately. Beware that polling a quiescent database forces a + * rotation per call. See core.c for the full contract. + */ +platform_status +core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns); + platform_status core_unmount(core_handle *spl); @@ -300,6 +431,7 @@ core_config_init(core_config *trunk_cfg, uint64 queue_scale_percent, uint64 prefetch_budget, bool32 use_log, + uint64 checkpoint_log_size_bytes, bool32 use_stats, bool32 verbose_logging, platform_log_handle *log_handle); diff --git a/src/log.h b/src/log.h index 325d0764..09ce8afe 100644 --- a/src/log.h +++ b/src/log.h @@ -11,25 +11,64 @@ #include "cache.h" #include "data_internal.h" +#include "iterator.h" typedef struct log_handle log_handle; typedef struct log_iterator log_iterator; typedef struct log_config log_config; +/* + * The on-disk head of one mini-allocator-backed log stream: the data head + * (where replay begins), the metadata head (which owns the stream's extents), + * and a per-stream magic that validates its pages. Fixed at creation; a + * higher-level checkpoint record stores it to later find the stream for replay + * or reclaim it via log_dec_ref(). + */ +typedef struct log_head { + uint64 addr; // data head: first log page, where replay begins + uint64 meta_addr; // mini-allocator metadata head; owns the stream's extents + uint64 magic; // per-stream magic; validates the stream's pages +} log_head; + typedef int (*log_write_fn)(log_handle *log, key tuple_key, message data, - uint64 generation); -typedef void (*log_release_fn)(log_handle *log); -typedef uint64 (*log_addr_fn)(log_handle *log); -typedef uint64 (*log_magic_fn)(log_handle *log); + uint64 memtable_generation, + uint64 leaf_generation); +/* + * Finalize and retire the log stream, terminally. Finalizes the current + * append pages into checksummed, immutable pages, releases in-memory + * resources, and frees the handle (which is invalid afterward). + * + * The caller must exclude concurrent log_write() and log_seal() calls. seal() + * itself issues no writeback or durable barrier: to make the sealed pages + * durable, the caller takes the cache writeback fence + a durable barrier + * afterward. The stream's head is fixed at creation and obtained then via + * log_get_head(), so seal needs no out-parameter; the caller frees the on-disk + * extents later via log_dec_ref(). + */ +typedef platform_status (*log_seal_fn)(log_handle *log); +/* + * The stream's durable head, fixed at creation. The caller records it + * (e.g. in the superblock) as soon as the log is created, so that a crash + * mid-stream can find the stream for replay. + */ +typedef log_head (*log_head_fn)(log_handle *log); +/* + * Bytes appended to the stream so far, so a caller can decide when to retire + * it. Excludes the implementation's fixed per-stream overhead: a stream that + * has had nothing written to it reports 0, which keeps a size-triggered policy + * from firing on a brand-new stream no matter how small its threshold. A + * conservative measure otherwise -- space is counted as it is reserved, so this + * rounds up to whatever allocation unit the implementation uses. + */ +typedef uint64 (*log_size_fn)(log_handle *log); typedef struct log_ops { - log_write_fn write; - log_release_fn release; - log_addr_fn addr; - log_addr_fn meta_addr; - log_magic_fn magic; + log_write_fn write; + log_seal_fn seal; + log_head_fn head; + log_size_fn size; } log_ops; // to sub-class log, make a log_handle your first field @@ -38,34 +77,120 @@ struct log_handle { }; static inline int -log_write(log_handle *log, key tuple_key, message data, uint64 generation) +log_write(log_handle *log, + key tuple_key, + message data, + uint64 memtable_generation, + uint64 leaf_generation) { - return log->ops->write(log, tuple_key, data, generation); + return log->ops->write( + log, tuple_key, data, memtable_generation, leaf_generation); } -static inline void -log_release(log_handle *log) +/* + * Finalize and retire the log, freeing the handle. See log_seal_fn for the + * required exclusion and durability ordering; the handle is invalid after this + * returns. Capture the head via log_get_head() beforehand (it is fixed at + * creation). + */ +static inline platform_status +log_seal(log_handle *log) { - log->ops->release(log); + return log->ops->seal(log); } -static inline uint64 -log_addr(log_handle *log) +/* The stream's durable head (fixed at creation). See log_head_fn. */ +static inline log_head +log_get_head(log_handle *log) { - return log->ops->addr(log); + return log->ops->head(log); } +/* Bytes the stream currently occupies on disk. See log_size_fn. */ static inline uint64 -log_meta_addr(log_handle *log) +log_get_size(log_handle *log) { - return log->ops->meta_addr(log); + return log->ops->size(log); } -static inline uint64 -log_magic(log_handle *log) +/* + * A log_handle is created by the concrete log implementation -- e.g. + * shard_log_create() -- and then driven through the abstract ops above; it is + * freed by log_seal(). + */ + +/* + * Release a sealed log identified by its log_head: drop the reference its + * metadata head holds, freeing the stream's on-disk extents. Takes no handle + * -- the handle was freed by log_seal(); the caller retained only the head + * (log_get_head(), captured at creation). + */ +void +log_dec_ref(cache *cc, const log_head *head); + +/* + * ---- Abstract log iteration ---- + * + * A log_iterator reads a sealed log's records in generation order (used by + * crash recovery to replay a stream onto the durable root). It is a generic + * iterator (curr/can_next/next, via the embedded `super`) plus the log-specific + * ops below. To sub-class, make a log_iterator your first field. + */ +typedef void (*log_iterator_curr_generations_fn)(log_iterator *itor, + uint64 *memtable_generation, + uint64 *leaf_generation); +typedef void (*log_iterator_deinit_fn)(log_iterator *itor); + +typedef struct log_iterator_ops { + log_iterator_curr_generations_fn curr_generations; + log_iterator_deinit_fn deinit; +} log_iterator_ops; + +struct log_iterator { + iterator super; // generic iteration: curr / can_next / next + const log_iterator_ops *ops; +}; + +/* + * A log_iterator is created by the concrete log implementation -- e.g. + * shard_log_iterator_create() -- which fills in the ops below; callers then + * drive it through this abstract interface and free it with + * log_iterator_deinit(). + */ + +/* Whether a current record exists (safe to call curr / curr_generations). */ +static inline bool32 +log_iterator_can_next(log_iterator *itor) +{ + return iterator_can_next(&itor->super); +} + +/* The current record's key and message. Requires a current record. */ +static inline void +log_iterator_curr(log_iterator *itor, key *curr_key, message *msg) +{ + iterator_curr(&itor->super, curr_key, msg); +} + +/* The current record's generation metadata. Requires a current record. */ +static inline void +log_iterator_curr_generations(log_iterator *itor, + uint64 *memtable_generation, + uint64 *leaf_generation) { - return log->ops->magic(log); + itor->ops->curr_generations(itor, memtable_generation, leaf_generation); } -log_handle * -log_create(cache *cc, log_config *cfg, platform_heap_id hid); +/* Advance to the next record. */ +static inline platform_status +log_iterator_next(log_iterator *itor) +{ + return iterator_next(&itor->super); +} + +/* Free the iterator and its resources; the handle is invalid afterward. */ +static inline void +log_iterator_deinit(log_iterator *itor) +{ + itor->ops->deinit(itor); +} diff --git a/src/memtable.c b/src/memtable.c index 41482e68..13793514 100644 --- a/src/memtable.c +++ b/src/memtable.c @@ -80,6 +80,27 @@ memtable_end_insert(memtable_context *ctxt) batch_rwlock_unget(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); } +/* + * Exclude all inserts, including one that has already acquired its shared + * insert lock, so the caller can safely mutate generation state. Used by + * memtable_force_rotation(); the normal rotation path instead upgrades its own + * insert lock via memtable_try_begin_insert_rotation(). Pair with + * memtable_unblock_inserts(). + */ +static void +memtable_block_inserts(memtable_context *ctxt) +{ + batch_rwlock_get(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); + batch_rwlock_claim_loop(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); + batch_rwlock_lock(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); +} + +static void +memtable_unblock_inserts(memtable_context *ctxt) +{ + batch_rwlock_full_unlock(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); +} + static inline bool32 memtable_try_begin_insert_rotation(memtable_context *ctxt) { @@ -97,20 +118,6 @@ memtable_end_insert_rotation(memtable_context *ctxt) batch_rwlock_unclaim(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); } -static inline void -memtable_begin_raw_rotation(memtable_context *ctxt) -{ - batch_rwlock_get(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); - batch_rwlock_claim_loop(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); - batch_rwlock_lock(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); -} - -static inline void -memtable_end_raw_rotation(memtable_context *ctxt) -{ - batch_rwlock_full_unlock(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); -} - void memtable_begin_lookup(memtable_context *ctxt) { @@ -193,6 +200,14 @@ memtable_maybe_rotate_and_begin_insert(memtable_context *ctxt, current_generation); memtable_mark_empty(ctxt); + /* + * Still holding the insert lock exclusively: all in-flight inserts + * (and their log writes) have drained and none can start. This is + * the one safe point to swap the checkpoint's live log. + */ + if (ctxt->rotate != NULL) { + ctxt->rotate(ctxt->process_ctxt, current_generation); + } memtable_end_insert_rotation(ctxt); memtable_end_insert(ctxt); memtable_process(ctxt, current_generation); @@ -295,9 +310,9 @@ memtable_mark_incorporation_failed(memtable *mt, platform_status status) } uint64 -memtable_force_finalize(memtable_context *ctxt) +memtable_force_rotation(memtable_context *ctxt) { - memtable_begin_raw_rotation(ctxt); + memtable_block_inserts(ctxt); uint64 generation = ctxt->generation; uint64 mt_no = generation % ctxt->cfg.max_memtables; @@ -308,7 +323,22 @@ memtable_force_finalize(memtable_context *ctxt) <= ctxt->cfg.max_memtables); memtable_mark_empty(ctxt); - memtable_end_raw_rotation(ctxt); + /* + * Inserts are still excluded, exactly as in the natural (fullness-triggered) + * rotation: all in-flight inserts (and their log writes) have drained and + * none can start. See memtable_maybe_rotate_and_begin_insert() for why this + * is the one safe point to swap the checkpoint's live log. + */ + if (ctxt->rotate != NULL) { + ctxt->rotate(ctxt->process_ctxt, current_generation); + } + + memtable_unblock_inserts(ctxt); + + // Dispatch the rotated memtable, outside the critical section, exactly as + // the natural rotation path does. + memtable_process(ctxt, current_generation); + return current_generation; } @@ -357,12 +387,14 @@ memtable_deinit(cache *cc, memtable *mt) } platform_status -memtable_context_init(memtable_context *ctxt, - platform_heap_id hid, - cache *cc, - memtable_config *cfg, - process_fn process, - void *process_ctxt) +memtable_context_init_at_generation(memtable_context *ctxt, + platform_heap_id hid, + cache *cc, + memtable_config *cfg, + process_fn rotate, + process_fn process, + void *process_ctxt, + uint64 first_generation) { platform_status rc; ZERO_CONTENTS(ctxt); @@ -370,14 +402,22 @@ memtable_context_init(memtable_context *ctxt, ctxt->cfg = *cfg; ctxt->hid = hid; - if (MAX_MEMTABLES < cfg->max_memtables) { - platform_error_log("Configured number of memtables (%lu) exceeds max " - "supported memtables (%d)\n", + if (cfg->max_memtables == 0 || MAX_MEMTABLES < cfg->max_memtables) { + platform_error_log("Configured number of memtables (%lu) must be " + "between 1 and %d\n", cfg->max_memtables, MAX_MEMTABLES); return STATUS_BAD_PARAM; } + if (first_generation > (uint64)-1 - cfg->max_memtables) { + platform_error_log("First memtable generation (%lu) leaves too little " + "space for %lu memtable slots\n", + first_generation, + cfg->max_memtables); + return STATUS_BAD_PARAM; + } + rc = platform_mutex_init( &ctxt->incorporation_mutex, platform_get_module_id(), hid); if (!SUCCESS(rc)) { @@ -394,23 +434,47 @@ memtable_context_init(memtable_context *ctxt, batch_rwlock_init(&ctxt->rwlock); - for (uint64 mt_no = 0; mt_no < cfg->max_memtables; mt_no++) { - uint64 generation = mt_no; + for (uint64 generation_offset = 0; generation_offset < cfg->max_memtables; + generation_offset++) + { + uint64 generation = first_generation + generation_offset; + uint64 mt_no = generation % cfg->max_memtables; memtable_init(&ctxt->mt[mt_no], cc, cfg, generation); } - ctxt->generation = 0; - ctxt->generation_to_incorporate = 0; - ctxt->generation_retired = (uint64)-1; + ctxt->generation = first_generation; + ctxt->generation_to_incorporate = first_generation; + /* + * A fresh database has no incorporated generation. The UINT64_MAX + * sentinel preserves the existing unsigned generation-ring arithmetic. + * Otherwise, the checkpoint has incorporated every generation before + * first_generation. + */ + ctxt->generation_retired = + first_generation == 0 ? (uint64)-1 : first_generation - 1; ctxt->is_empty = TRUE; + ctxt->rotate = rotate; ctxt->process = process; ctxt->process_ctxt = process_ctxt; return STATUS_OK; } +platform_status +memtable_context_init(memtable_context *ctxt, + platform_heap_id hid, + cache *cc, + memtable_config *cfg, + process_fn rotate, + process_fn process, + void *process_ctxt) +{ + return memtable_context_init_at_generation( + ctxt, hid, cc, cfg, rotate, process, process_ctxt, 0); +} + void memtable_context_deinit(memtable_context *ctxt) { diff --git a/src/memtable.h b/src/memtable.h index 7c82aadd..443cd5cf 100644 --- a/src/memtable.h +++ b/src/memtable.h @@ -124,9 +124,20 @@ typedef struct memtable_context { task_system *ts; platform_heap_id *hid; + /* + * Optional callback invoked inside the rotation critical section (insert + * lock held exclusively), after a memtable is finalized and the generation + * is advanced, before the lock is released. A checkpoint uses this to swap + * in a new live log with the guarantee that no insert can be mid-log_write. + * Receives process_ctxt and the just-finalized generation. NULL disables. + */ + process_fn rotate; + /* Process a rotated memtable _outside the critical section of the rotation. + */ process_fn process; void *process_ctxt; + // batch distributed read/write locks protect the generation and // generation_retired counters batch_rwlock rwlock; @@ -194,8 +205,19 @@ memtable_mark_incorporation_failed(memtable *mt, platform_status status); const char * memtable_state_string(memtable_state state); +/* + * Rotate the current memtable now, regardless of fullness. Performs the same + * sequence as the natural (fullness-triggered) rotation: finalize the memtable, + * advance the generation, invoke the rotate callback under insert exclusion, + * then -- once inserts are unblocked -- invoke the process callback to dispatch + * the rotated memtable. Returns the finalized generation. + * + * Callers therefore need do nothing further: whatever the rotate callback + * started (e.g. core's checkpoint log cut) is resolved by the process callback, + * just as it is for a natural rotation. + */ uint64 -memtable_force_finalize(memtable_context *ctxt); +memtable_force_rotation(memtable_context *ctxt); void memtable_init(memtable *mt, cache *cc, memtable_config *cfg, uint64 generation); @@ -208,9 +230,29 @@ memtable_context_init(memtable_context *ctxt, platform_heap_id hid, cache *cc, memtable_config *cfg, + process_fn rotate, process_fn process, void *process_ctxt); +/* + * Initialize the reusable memtable ring at first_generation. Recovery passes + * the checkpoint's incorporated generation plus one; the trunk is understood + * to have incorporated all earlier generations. In particular, slot + * first_generation % max_memtables is the active memtable and the other slots + * represent the following logical generations. first_generation == 0 is the + * fresh-database case and has no incorporated generation. A caller must not + * wrap a checkpoint generation of UINT64_MAX into zero. + */ +platform_status +memtable_context_init_at_generation(memtable_context *ctxt, + platform_heap_id hid, + cache *cc, + memtable_config *cfg, + process_fn rotate, + process_fn process, + void *process_ctxt, + uint64 first_generation); + void memtable_context_deinit(memtable_context *ctxt); diff --git a/src/mini_allocator.c b/src/mini_allocator.c index 1099641f..a4f328d6 100644 --- a/src/mini_allocator.c +++ b/src/mini_allocator.c @@ -198,7 +198,6 @@ mini_full_lock_meta_tail(mini_allocator *mini) static void mini_full_unlock_meta_page(mini_allocator *mini, page_handle *meta_page) { - cache_mark_dirty(mini->cc, meta_page); cache_unlock(mini->cc, meta_page); cache_unclaim(mini->cc, meta_page); cache_unget(mini->cc, meta_page); @@ -761,35 +760,78 @@ mini_deinit(cache *cc, uint64 meta_head, page_type type) *----------------------------------------------------------------------------- * mini_for_each_meta_page -- * - * Calls func on each meta_page in the mini_allocator. + * Walks the meta_page chain of the mini_allocator, calling func on each + * meta_page. + * + * If before is non-NULL, it is called once per distinct meta extent -- + * before that extent's first page is cache_get()'d -- with just the + * extent's address. This lets a caller that does not yet have allocator + * ownership of these pages (e.g. crash recovery, rebuilding the refcount + * table from scratch) establish it first, before cache_get()'s debug + * allocation checks run. Ordinary callers, which already own everything + * they are walking, pass NULL. + * + * func and before may fail; the walk stops and returns the first failure. * * Results: - * None + * STATUS_OK, the first failure from before/func, or STATUS_IO_ERROR if a + * meta page could not be read. * * Side effects: - * func may store output in arg. + * func/before may store output in arg. *----------------------------------------------------------------------------- */ -typedef void (*mini_for_each_meta_page_fn)(cache *cc, - page_type type, - page_handle *meta_page, - void *arg); +typedef platform_status (*mini_meta_extent_fn)(cache *cc, + page_type type, + uint64 extent_addr, + void *arg); -static void +typedef platform_status (*mini_for_each_meta_page_fn)(cache *cc, + page_type type, + page_handle *meta_page, + void *arg); + +static platform_status mini_for_each_meta_page(cache *cc, uint64 meta_head, page_type type, + mini_meta_extent_fn before, mini_for_each_meta_page_fn func, void *arg) { - uint64 meta_addr = meta_head; + uint64 meta_addr = meta_head; + uint64 extent_size = cache_extent_size(cc); + uint64 prior_extent = (uint64)-1; + while (meta_addr != 0) { + if (before != NULL) { + uint64 extent_addr = meta_addr - meta_addr % extent_size; + if (extent_addr != prior_extent) { + platform_status rc = before(cc, type, extent_addr, arg); + if (!SUCCESS(rc)) { + return rc; + } + prior_extent = extent_addr; + } + } + page_handle *meta_page = cache_get(cc, meta_addr, TRUE, type); - func(cc, type, meta_page, arg); - meta_addr = mini_get_next_meta_addr(meta_page); + if (meta_page == NULL) { + return STATUS_IO_ERROR; + } + + platform_status rc = + func == NULL ? STATUS_OK : func(cc, type, meta_page, arg); + uint64 next_meta_addr = + SUCCESS(rc) ? mini_get_next_meta_addr(meta_page) : 0; cache_unget(cc, meta_page); + if (!SUCCESS(rc)) { + return rc; + } + meta_addr = next_meta_addr; } + return STATUS_OK; } /* mini_for_each(): call a function on each allocated extent in the @@ -805,7 +847,7 @@ typedef struct for_each_func { void *arg; } for_each_func; -static void +static platform_status mini_for_each_meta_page_func(cache *cc, page_type type, page_handle *meta_page, @@ -822,6 +864,7 @@ mini_for_each_meta_page_func(cache *cc, fef->arg); entry = next_entry(entry); } + return STATUS_OK; } static void @@ -833,7 +876,7 @@ mini_for_each(cache *cc, { for_each_func fef = {func, out}; mini_for_each_meta_page( - cc, meta_head, type, mini_for_each_meta_page_func, &fef); + cc, meta_head, type, NULL, mini_for_each_meta_page_func, &fef); } @@ -917,6 +960,270 @@ mini_prefetch(cache *cc, page_type type, uint64 meta_head) mini_for_each(cc, meta_head, type, mini_prefetch_extent, NULL); } +/* + * ----------------------------------------------------------------------------- + * mini_recover_allocations -- crash-recovery allocator-reference rebuild. + * ----------------------------------------------------------------------------- + */ + +static platform_status +mini_recovery_corruption(const char *reason, uint64 addr) +{ + platform_error_log( + "Malformed mini allocator metadata: %s (addr=%lu).\n", reason, addr); + return STATUS_INVALID_STATE; +} + +static bool32 +mini_recovery_valid_geometry(const allocator_config *cfg) +{ + if (cfg == NULL || cfg->io_cfg == NULL || cfg->capacity == 0 + || cfg->io_cfg->page_size == 0 || cfg->io_cfg->extent_size == 0 + || cfg->io_cfg->page_size < offsetof(mini_meta_hdr, entry_buffer) + || cfg->io_cfg->extent_size < cfg->io_cfg->page_size + || cfg->io_cfg->extent_size % cfg->io_cfg->page_size != 0 + || cfg->capacity % cfg->io_cfg->extent_size != 0) + { + return FALSE; + } + + return TRUE; +} + +static bool32 +mini_recovery_valid_page_addr(const allocator_config *cfg, uint64 addr) +{ + uint64 page_size = cfg->io_cfg->page_size; + + return addr != 0 && addr % page_size == 0 + && addr <= cfg->capacity - page_size; +} + +static bool32 +mini_recovery_valid_extent_addr(const allocator_config *cfg, uint64 addr) +{ + uint64 extent_size = cfg->io_cfg->extent_size; + + return addr != 0 && addr % extent_size == 0 + && addr <= cfg->capacity - extent_size; +} + +static platform_status +mini_recovery_validate_meta_header(const mini_meta_hdr *hdr, + uint64 page_size, + uint64 expected_prev, + uint64 meta_addr) +{ + uint64 first_entry_offset = offsetof(mini_meta_hdr, entry_buffer); + uint64 max_entries = (page_size - first_entry_offset) / sizeof(meta_entry); + if (hdr->prev_meta_addr != expected_prev) { + return mini_recovery_corruption("metadata prev link is not reciprocal", + meta_addr); + } + + if (hdr->num_entries > max_entries) { + return mini_recovery_corruption("metadata entry count exceeds page", + meta_addr); + } + + uint64 expected_pos = + first_entry_offset + (uint64)hdr->num_entries * sizeof(meta_entry); + if (hdr->pos != expected_pos) { + return mini_recovery_corruption("metadata entry position is invalid", + meta_addr); + } + + return STATUS_OK; +} + +static platform_status +mini_recovery_validate_next_meta_addr(const allocator_config *cfg, + uint64 meta_addr, + uint64 next_meta_addr) +{ + if (next_meta_addr == 0) { + return STATUS_OK; + } + + if (!mini_recovery_valid_page_addr(cfg, next_meta_addr)) { + return mini_recovery_corruption("metadata next link is not a page", + next_meta_addr); + } + + uint64 extent_size = cfg->io_cfg->extent_size; + uint64 page_size = cfg->io_cfg->page_size; + uint64 offset = meta_addr % extent_size; + if (offset == extent_size - page_size) { + if (next_meta_addr % extent_size != 0) { + return mini_recovery_corruption( + "metadata extent transition does not start at an extent base", + next_meta_addr); + } + } else if (next_meta_addr != meta_addr + page_size) { + return mini_recovery_corruption( + "metadata pages do not advance contiguously within an extent", + next_meta_addr); + } + + return STATUS_OK; +} + +typedef struct mini_recover_state { + allocator *al; + allocator_config *cfg; + uint64 expected_prev; + uint64 page_count; + uint64 max_meta_pages; +} mini_recover_state; + +/* + * The "before" hook mini_for_each_meta_page() calls once per distinct meta + * extent, before that extent's first page is cache_get()'d. Recording the + * reference here -- ahead of the read -- is what lets a rebuilding allocator + * (which does not yet consider this extent allocated) satisfy cache_get()'s + * debug allocation check. + */ +static platform_status +mini_recover_visit_meta_extent(cache *cc, + page_type type, + uint64 extent_addr, + void *arg) +{ + mini_recover_state *state = (mini_recover_state *)arg; + + if (!mini_recovery_valid_extent_addr(state->cfg, extent_addr)) { + return mini_recovery_corruption("metadata extent is out of range", + extent_addr); + } + return allocator_recovery_record_reference(state->al, extent_addr, type); +} + +/* + * The "func" hook mini_for_each_meta_page() calls once per meta page, with the + * page held. Validates the page and every data-extent entry on it before + * trusting any of it, and records a reference for each data extent found (its + * type is self-describing on disk, unlike a meta page's). + */ +static platform_status +mini_recover_visit_meta_page(cache *cc, + page_type type, + page_handle *meta_page, + void *arg) +{ + mini_recover_state *state = (mini_recover_state *)arg; + uint64 meta_addr = meta_page->disk_addr; + + if (state->page_count == state->max_meta_pages) { + return mini_recovery_corruption("metadata chain exceeds disk pages", + meta_addr); + } + state->page_count++; + + mini_meta_hdr *hdr = (mini_meta_hdr *)meta_page->data; + platform_status rc = mini_recovery_validate_meta_header( + hdr, state->cfg->io_cfg->page_size, state->expected_prev, meta_addr); + if (!SUCCESS(rc)) { + return rc; + } + + rc = mini_recovery_validate_next_meta_addr( + state->cfg, meta_addr, hdr->next_meta_addr); + if (!SUCCESS(rc)) { + return rc; + } + + meta_entry *entry = first_entry(meta_page); + for (uint64 entry_no = 0; entry_no < hdr->num_entries; entry_no++) { + uint64 batch = meta_entry_batch(entry); + page_type extent_type = meta_entry_type(entry); + uint64 extent_number = + entry->packed >> (META_ENTRY_BATCH_BITS + META_ENTRY_TYPE_BITS); + + if (batch >= MINI_MAX_BATCHES || extent_type < PAGE_TYPE_FIRST + || extent_type >= NUM_PAGE_TYPES || extent_number == 0 + || extent_number > UINT64_MAX / state->cfg->io_cfg->extent_size) + { + return mini_recovery_corruption("metadata extent entry is invalid", + meta_addr); + } + + uint64 extent_addr = extent_number * state->cfg->io_cfg->extent_size; + if (!mini_recovery_valid_extent_addr(state->cfg, extent_addr)) { + return mini_recovery_corruption("metadata extent is out of range", + extent_addr); + } + + rc = allocator_recovery_record_reference( + state->al, extent_addr, extent_type); + if (!SUCCESS(rc)) { + return rc; + } + + entry = next_entry(entry); + } + + state->expected_prev = meta_addr; + return STATUS_OK; +} + +/* + * mini_recover_allocations -- + * + * Rebuild allocator references for a finalized mini allocator without + * trusting allocator refcounts -- the discovery primitive crash recovery + * uses to reconstruct them. Records one reference for every metadata + * extent and every data-extent entry found in the on-disk mini metadata + * stream (data entries are not deduplicated: repeated entries mean + * repeated references, and are recorded exactly as often as they occur). + * + * meta_type is the page type of meta_head's own chain; each data extent's + * type is read from its own metadata entry. + * + * Validates the metadata-page chain, page-header bounds, page types, + * batches, and extent addresses before using them. A malformed on-disk + * stream returns STATUS_INVALID_STATE. This function performs no writes + * and does not use allocator refcounts to decide what to traverse. + * + * This is a physical enumeration only. Recovering logical reference + * multiplicity, and deduplicating references shared by distinct mini + * allocator roots, remains the responsibility of the higher-level + * trunk/log recovery walker. + */ +platform_status +mini_recover_allocations(cache *cc, uint64 meta_head, page_type meta_type) +{ + if (cc == NULL || meta_type < PAGE_TYPE_FIRST || meta_type >= NUM_PAGE_TYPES) + { + return STATUS_BAD_PARAM; + } + + allocator *al = cache_get_allocator(cc); + if (al == NULL) { + return STATUS_BAD_PARAM; + } + allocator_config *cfg = allocator_get_config(al); + if (!mini_recovery_valid_geometry(cfg)) { + return STATUS_BAD_PARAM; + } + if (!mini_recovery_valid_page_addr(cfg, meta_head)) { + return mini_recovery_corruption("metadata head is not a page", meta_head); + } + + mini_recover_state state = {.al = al, + .cfg = cfg, + .expected_prev = 0, + .page_count = 0, + .max_meta_pages = + cfg->capacity / cfg->io_cfg->page_size}; + + return mini_for_each_meta_page(cc, + meta_head, + meta_type, + mini_recover_visit_meta_extent, + mini_recover_visit_meta_page, + &state); +} + /* *----------------------------------------------------------------------------- * mini_meta_cursor -- cursor over a mini_allocator's extent entries. @@ -1080,7 +1387,7 @@ space_use_add_extent(cache *cc, page_type type, uint64 extent_addr, void *out) *sum += cache_extent_size(cc); } -static void +static platform_status space_use_add_meta_page(cache *cc, page_type type, page_handle *meta_page, @@ -1088,6 +1395,7 @@ space_use_add_meta_page(cache *cc, { uint64 *sum = (uint64 *)out; *sum += cache_page_size(cc); + return STATUS_OK; } uint64 @@ -1096,7 +1404,7 @@ mini_space_use_bytes(cache *cc, uint64 meta_head, page_type type) uint64 total = 0; mini_for_each(cc, meta_head, type, space_use_add_extent, &total); mini_for_each_meta_page( - cc, meta_head, type, space_use_add_meta_page, &total); + cc, meta_head, type, NULL, space_use_add_meta_page, &total); return total; } diff --git a/src/mini_allocator.h b/src/mini_allocator.h index 1a7995b7..5407530f 100644 --- a/src/mini_allocator.h +++ b/src/mini_allocator.h @@ -113,6 +113,33 @@ mini_unblock_dec_ref(cache *cc, uint64 meta_head); void mini_prefetch(cache *cc, page_type type, uint64 meta_head); +/* + * mini_recover_allocations -- + * + * Rebuild allocator references for a finalized mini allocator without + * trusting allocator refcounts. This is the discovery primitive crash + * recovery uses to reconstruct them: it records one allocator reference + * (via allocator_recovery_record_reference()) for every metadata extent + * and for every data-extent entry in the on-disk mini metadata stream. + * Data entries are deliberately not deduplicated: repeated entries + * represent repeated references in the mini allocator and are recorded + * exactly as often as they occur. meta_type is the page type of + * meta_head's own chain; each data extent's type is read from its own + * metadata entry, so it is not a parameter here. + * + * Validates the metadata-page chain, page-header bounds, page types, + * batches, and extent addresses before using them. A malformed on-disk + * stream returns STATUS_INVALID_STATE. This function performs no writes + * and does not use allocator refcounts to decide what to traverse. + * + * This is a physical enumeration only. Recovering logical reference + * multiplicity, and deduplicating references shared by distinct mini + * allocator roots, remains the responsibility of the higher-level + * trunk/log recovery walker. + */ +platform_status +mini_recover_allocations(cache *cc, uint64 meta_head, page_type meta_type); + /* * mini_meta_cursor: a non-blocking cursor over the extent entries of a * finalized mini_allocator. Entries from all batches are interleaved in diff --git a/src/platform_linux/laio.c b/src/platform_linux/laio.c index 87e6c3f5..6988514b 100644 --- a/src/platform_linux/laio.c +++ b/src/platform_linux/laio.c @@ -213,6 +213,35 @@ laio_write(io_handle *ioh, void *buf, uint64 bytes, uint64 addr) return STATUS_IO_ERROR; } +/* + *-------------------------------------------------------------------------- + * laio_durable_barrier -- + * + * Make completed writes durable. Waiting for asynchronous I/O only tells us + * that the kernel accepted the writes; fdatasync() supplies the persistence + * ordering needed by checkpoint publication on buffered files. + *-------------------------------------------------------------------------- + */ +static platform_status +laio_durable_barrier(io_handle *ioh) +{ + laio_handle *io = (laio_handle *)ioh; + int ret; + + do { + ret = fdatasync(io->fd); + } while (ret != 0 && errno == EINTR); + + if (ret == 0) { + return STATUS_OK; + } + + int saved_errno = errno; + platform_error_log("laio_durable_barrier: fdatasync failed: %s\n", + strerror(saved_errno)); + return CONST_STATUS(saved_errno); +} + /* * Accessor method: Return opaque handle to IO-context setup by io_setup(). */ @@ -674,6 +703,7 @@ static io_ops laio_ops = { .async_state_init = laio_async_state_init, .cleanup = laio_cleanup, .wait_all = laio_wait_all, + .durable_barrier = laio_durable_barrier, .print_stats = laio_print_stats, .reset_stats = laio_reset_stats, }; diff --git a/src/platform_linux/platform_io.h b/src/platform_linux/platform_io.h index 1d96bf1a..cb60e861 100644 --- a/src/platform_linux/platform_io.h +++ b/src/platform_linux/platform_io.h @@ -78,6 +78,7 @@ typedef platform_status (*io_async_state_init_fn)(io_async_state *state, typedef void (*io_cleanup_fn)(io_handle *io, uint64 count); typedef void (*io_wait_all_fn)(io_handle *io); +typedef platform_status (*io_durable_barrier_fn)(io_handle *io); typedef void (*io_register_thread_fn)(io_handle *io); typedef void (*io_deregister_thread_fn)(io_handle *io); typedef bool32 (*io_max_latency_elapsed_fn)(io_handle *io, timestamp ts); @@ -95,6 +96,7 @@ typedef struct io_ops { io_async_state_init_fn async_state_init; io_cleanup_fn cleanup; io_wait_all_fn wait_all; + io_durable_barrier_fn durable_barrier; io_register_thread_fn register_thread; io_deregister_thread_fn deregister_thread; io_max_latency_elapsed_fn max_latency_elapsed; @@ -197,13 +199,34 @@ io_cleanup(io_handle *io, uint64 count) return io->ops->cleanup(io, count); } -// Guarantees all in-flight IOs are complete before return +// Wait until we have seen each process's I/O be quiescent. +// +// This means that all I/Os that were in-flight at the start of this call are +// guaranteed to have finished before this call returns. +// +// But I/Os that are issued after this call starts may not be complete when +// this call returns. static inline void io_wait_all(io_handle *io) { return io->ops->wait_all(io); } +/* + *-------------------------------------------------------------------------- + * io_durable_barrier + * + * Ensure that writes which completed before this call survive a power loss. + * This is deliberately distinct from io_wait_all(), which only waits for + * asynchronous I/O completion. + *-------------------------------------------------------------------------- + */ +static inline platform_status +io_durable_barrier(io_handle *io) +{ + return io->ops->durable_barrier(io); +} + static inline void io_register_thread(io_handle *io) { diff --git a/src/rc_allocator.c b/src/rc_allocator.c index 70257dc5..2bedb5c0 100644 --- a/src/rc_allocator.c +++ b/src/rc_allocator.c @@ -17,13 +17,17 @@ #include "platform_typed_alloc.h" #include "poison.h" -#define RC_ALLOCATOR_META_PAGE_CSUM_SEED (2718281828) - /* - * Base offset from where the allocator starts. Currently hard coded to 0. + * The refcount map is the allocator's only durable structure. It lives in a + * fixed run of extents starting at extent 1 (extent 0 is reserved for the + * superblock). Whether the persisted map is trustworthy is recorded by the + * superblock (allocation_state_addr), which the caller consults to choose a + * clean vs recovery mount -- the allocator itself no longer keeps a + * clean-state record or a bootstrap meta page. */ -#define RC_ALLOCATOR_BASE_OFFSET (0) +/* Extent 0 holds the superblock; the refcount map begins at extent 1. */ +#define RC_ALLOCATOR_REFCOUNT_MAP_EXTENT (1) /* A predicate defining whether to trace allocations/ref-count changes * on a given address. @@ -35,168 +39,6 @@ */ #define SHOULD_TRACE(addr) (0) // Do not trace anything -/* - *------------------------------------------------------------------------------ - * Function declarations and virtual trampolines - *------------------------------------------------------------------------------ - */ - -// allocator.h functions - -allocator_config * -rc_allocator_get_config(rc_allocator *al); - -allocator_config * -rc_allocator_get_config_virtual(allocator *a) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_get_config(al); -} - -platform_status -rc_allocator_alloc(rc_allocator *al, uint64 *addr, page_type type); - -platform_status -rc_allocator_alloc_virtual(allocator *a, uint64 *addr, page_type type) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_alloc(al, addr, type); -} - -refcount -rc_allocator_inc_ref(rc_allocator *al, uint64 addr); - -refcount -rc_allocator_inc_ref_virtual(allocator *a, uint64 addr) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_inc_ref(al, addr); -} - -refcount -rc_allocator_dec_ref(rc_allocator *al, uint64 addr, page_type type); - -refcount -rc_allocator_dec_ref_virtual(allocator *a, uint64 addr, page_type type) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_dec_ref(al, addr, type); -} - -refcount -rc_allocator_get_ref(rc_allocator *al, uint64 addr); - -refcount -rc_allocator_get_ref_virtual(allocator *a, uint64 addr) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_get_ref(al, addr); -} - -platform_status -rc_allocator_get_super_addr(rc_allocator *al, - allocator_root_id spl_id, - uint64 *addr); - -platform_status -rc_allocator_get_super_addr_virtual(allocator *a, - allocator_root_id spl_id, - uint64 *addr) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_get_super_addr(al, spl_id, addr); -} - -platform_status -rc_allocator_alloc_super_addr(rc_allocator *al, - allocator_root_id spl_id, - uint64 *addr); - -platform_status -rc_allocator_alloc_super_addr_virtual(allocator *a, - allocator_root_id spl_id, - uint64 *addr) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_alloc_super_addr(al, spl_id, addr); -} - -void -rc_allocator_remove_super_addr(rc_allocator *al, allocator_root_id spl_id); - -void -rc_allocator_remove_super_addr_virtual(allocator *a, allocator_root_id spl_id) -{ - rc_allocator *al = (rc_allocator *)a; - rc_allocator_remove_super_addr(al, spl_id); -} - -uint64 -rc_allocator_in_use(rc_allocator *al); - -uint64 -rc_allocator_in_use_virtual(allocator *a) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_in_use(al); -} - -uint64 -rc_allocator_get_capacity(rc_allocator *al); - -uint64 -rc_allocator_get_capacity_virtual(allocator *a) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_get_capacity(al); -} - -void -rc_allocator_assert_noleaks(rc_allocator *al); - -void -rc_allocator_assert_noleaks_virtual(allocator *a) -{ - rc_allocator *al = (rc_allocator *)a; - rc_allocator_assert_noleaks(al); -} - -void -rc_allocator_print_stats(rc_allocator *al); - -void -rc_allocator_print_stats_virtual(allocator *a) -{ - rc_allocator *al = (rc_allocator *)a; - rc_allocator_print_stats(al); -} - -void -rc_allocator_print_allocated(rc_allocator *al); - -void -rc_allocator_print_allocated_virtual(allocator *a) -{ - rc_allocator *al = (rc_allocator *)a; - rc_allocator_print_allocated(al); -} - -const static allocator_ops rc_allocator_ops = { - .get_config = rc_allocator_get_config_virtual, - .alloc = rc_allocator_alloc_virtual, - .inc_ref = rc_allocator_inc_ref_virtual, - .dec_ref = rc_allocator_dec_ref_virtual, - .get_ref = rc_allocator_get_ref_virtual, - .get_super_addr = rc_allocator_get_super_addr_virtual, - .alloc_super_addr = rc_allocator_alloc_super_addr_virtual, - .remove_super_addr = rc_allocator_remove_super_addr_virtual, - .in_use = rc_allocator_in_use_virtual, - .get_capacity = rc_allocator_get_capacity_virtual, - .assert_noleaks = rc_allocator_assert_noleaks_virtual, - .print_stats = rc_allocator_print_stats_virtual, - .print_allocated = rc_allocator_print_allocated_virtual, -}; - /* * Helper methods */ @@ -223,97 +65,109 @@ rc_allocator_extent_number(rc_allocator *al, uint64 addr) return (addr / al->cfg->io_cfg->extent_size); } -static checksum128 -rc_allocator_meta_page_checksum(const rc_allocator_meta_page *meta_page) +static uint64 +rc_allocator_refcount_buffer_size(const allocator_config *cfg) { - return platform_checksum128(meta_page, - offsetof(rc_allocator_meta_page, checksum), - RC_ALLOCATOR_META_PAGE_CSUM_SEED); + uint64 buffer_size = cfg->extent_capacity * sizeof(refcount); + return ROUNDUP(buffer_size, cfg->io_cfg->page_size); } -static disk_geometry -rc_allocator_config_get_disk_geometry(allocator_config *cfg) +static uint64 +rc_allocator_refcount_extent_count(const allocator_config *cfg) { - return (disk_geometry){ - .disk_size = cfg->capacity, - .page_size = cfg->io_cfg->page_size, - .extent_size = cfg->io_cfg->extent_size, - }; + uint64 buffer_size = rc_allocator_refcount_buffer_size(cfg); + return (buffer_size + cfg->io_cfg->extent_size - 1) + / cfg->io_cfg->extent_size; } -static platform_status -rc_allocator_validate_disk_geometry(rc_allocator *al) +static uint64 +rc_allocator_reserved_extent_count(const allocator_config *cfg) { - disk_geometry geometry = al->meta_page->geometry; - - return rc_allocator_disk_geometry_matches_config(&geometry, al->cfg); + /* Extent 0 (superblock) + the refcount-map extents. No clean-state + * extents: superblock allocation_state_addr now records map validity. */ + return 1 + rc_allocator_refcount_extent_count(cfg); } -platform_status -rc_allocator_disk_geometry_matches_config(const disk_geometry *geometry, - const allocator_config *cfg) +static void +rc_allocator_record_allocated_extent(rc_allocator *al) { - if (geometry->disk_size != cfg->capacity - || geometry->page_size != cfg->io_cfg->page_size - || geometry->extent_size != cfg->io_cfg->extent_size) - { - platform_error_log( - "SplinterDB disk geometry does not match configuration: " - "disk=(disk_size=%lu, page_size=%lu, extent_size=%lu), " - "config=(disk_size=%lu, page_size=%lu, extent_size=%lu)\n", - geometry->disk_size, - geometry->page_size, - geometry->extent_size, - cfg->capacity, - cfg->io_cfg->page_size, - cfg->io_cfg->extent_size); - return STATUS_BAD_PARAM; + int64 curr_allocated = __sync_add_and_fetch(&al->stats.curr_allocated, 1); + int64 max_allocated = al->stats.max_allocated; + while (curr_allocated > max_allocated) { + __sync_bool_compare_and_swap( + &al->stats.max_allocated, max_allocated, curr_allocated); + max_allocated = al->stats.max_allocated; } - - return STATUS_OK; } -platform_status -rc_allocator_read_disk_geometry(const char *filename, disk_geometry *geometry) +/* + * Allocate the (zeroed) in-memory refcount buffer, if it is not already + * allocated. mkfs (rc_allocator_init), a clean load (rc_allocator_load_ + * refcounts), and a recovery rebuild (rc_allocator_recovery_begin) each ensure + * the buffer exists before populating it. Attach (rc_allocator_mount) + * deliberately does NOT: it leaves ref_count NULL so that using the allocator + * before a load/recovery faults immediately. Idempotent so it tolerates an + * allocator reused across mount cycles without re-attaching. + */ +static platform_status +rc_allocator_alloc_refcount_buffer(rc_allocator *al) { - return io_read_bootstrap( - filename, geometry, sizeof(*geometry), RC_ALLOCATOR_BASE_OFFSET); + if (al->ref_count != NULL) { + return STATUS_OK; + } + uint64 buffer_size = rc_allocator_refcount_buffer_size(al->cfg); + platform_status status = platform_buffer_init(&al->bh, buffer_size); + if (!SUCCESS(status)) { + platform_error_log("Failed to create buffer for ref counts\n"); + return status; + } + al->ref_count = platform_buffer_getaddr(&al->bh); + memset(al->ref_count, 0, buffer_size); + return STATUS_OK; } static platform_status -rc_allocator_init_meta_page(rc_allocator *al) +rc_allocator_init_refcounts(rc_allocator *al) { - /* - * To make it easier to do aligned i/o's we allocate the meta page to - * always be page size. In the future we can use the remaining space - * for some other reserved information we may want to persist as part - * of the meta page. - */ - platform_assert(sizeof(rc_allocator_meta_page) - <= al->cfg->io_cfg->page_size); - /* - * Ensure that the meta page and all the super blocks will fit in one - * extent. - */ - platform_assert((1 + RC_ALLOCATOR_MAX_ROOT_IDS) * al->cfg->io_cfg->page_size - <= al->cfg->io_cfg->extent_size); + platform_status status = rc_allocator_alloc_refcount_buffer(al); + if (!SUCCESS(status)) { + return status; + } + + uint64 reserved_extent_count = rc_allocator_reserved_extent_count(al->cfg); - al->meta_page = TYPED_ALIGNED_ZALLOC(al->heap_id, - al->cfg->io_cfg->page_size, - al->meta_page, - al->cfg->io_cfg->page_size); - if (al->meta_page == NULL) { - return STATUS_NO_MEMORY; + if (reserved_extent_count > al->cfg->extent_capacity) { + platform_error_log("Allocator needs %lu reserved extents, but its " + "configured capacity is only %lu extents.\n", + reserved_extent_count, + al->cfg->extent_capacity); + return STATUS_BAD_PARAM; } - memset(al->meta_page->splinters, - INVALID_ALLOCATOR_ROOT_ID, - sizeof(al->meta_page->splinters)); - al->meta_page->geometry = rc_allocator_config_get_disk_geometry(al->cfg); + memset(al->ref_count, 0, rc_allocator_refcount_buffer_size(al->cfg)); + + /* + * Extent 0 holds the superblock; the refcount map begins at extent 1. + * Reserve both so recovery never hands them out. + */ + for (uint64 extent_no = 0; extent_no < reserved_extent_count; extent_no++) { + platform_assert(al->ref_count[extent_no] == AL_FREE); + al->ref_count[extent_no] = AL_ONE_REF; + rc_allocator_record_allocated_extent(al); + } + /* Match a regular mount: the first post-recovery allocation scans at 0. */ + al->hand = 0; return STATUS_OK; } +static bool32 +rc_allocator_recovery_extent_is_reserved(const rc_allocator *al, + uint64 extent_no) +{ + return extent_no < rc_allocator_reserved_extent_count(al->cfg); +} + /* *---------------------------------------------------------------------- * rc_allocator_valid_config() -- @@ -344,6 +198,15 @@ rc_allocator_valid_config(allocator_config *cfg) return STATUS_BAD_PARAM; } + if (rc_allocator_reserved_extent_count(cfg) > cfg->extent_capacity) { + platform_error_log("Configured allocator has %lu extents, but needs %lu " + "reserved extents for metadata, refcounts, and clean " + "state.\n", + cfg->extent_capacity, + rc_allocator_reserved_extent_count(cfg)); + return STATUS_BAD_PARAM; + } + // Assert: Disk size == (page-size * #-of-pages) if (cfg->capacity != (cfg->io_cfg->page_size * cfg->page_capacity)) { platform_error_log("Configured disk size, %lu bytes, is not an integral" @@ -368,200 +231,157 @@ rc_allocator_valid_config(allocator_config *cfg) return rc; } - -/* - *---------------------------------------------------------------------- - * rc_allocator_[de]init -- - * - * [de]initialize an allocator - *---------------------------------------------------------------------- - */ platform_status -rc_allocator_init(rc_allocator *al, - allocator_config *cfg, - io_handle *io, - platform_heap_id hid, - platform_module_id mid) +rc_allocator_recovery_begin(rc_allocator *al) { - uint64 rc_extent_count; - uint64 addr; - platform_status rc; - platform_assert(al != NULL); - ZERO_CONTENTS(al); - al->super.ops = &rc_allocator_ops; - al->cfg = cfg; - al->io = io; - al->heap_id = hid; + return rc_allocator_init_refcounts(al); +} - rc = rc_allocator_valid_config(cfg); - if (!SUCCESS(rc)) { - return rc; +platform_status +rc_allocator_recovery_record_reference(rc_allocator *al, + uint64 extent_addr, + page_type type) +{ + if (al->map_is_valid) { + platform_error_log( + "Cannot acquire allocator extent once the refcount map is valid.\n"); + return STATUS_INVALID_STATE; } - rc = platform_mutex_init(&al->lock, mid, al->heap_id); - if (!SUCCESS(rc)) { - platform_error_log("Failed to init mutex for the allocator\n"); - return rc; - } - rc = rc_allocator_init_meta_page(al); - if (!SUCCESS(rc)) { - platform_error_log("Failed to init meta page for rc allocator\n"); - platform_mutex_destroy(&al->lock); - return rc; + uint64 extent_size = al->cfg->io_cfg->extent_size; + if (extent_addr >= al->cfg->capacity || extent_addr % extent_size != 0) { + platform_error_log("Invalid allocator recovery extent address %lu.\n", + extent_addr); + return STATUS_BAD_PARAM; } - // To ensure alignment always allocate in multiples of page size. - uint64 buffer_size = cfg->extent_capacity * sizeof(refcount); - buffer_size = ROUNDUP(buffer_size, cfg->io_cfg->page_size); - rc = platform_buffer_init(&al->bh, buffer_size); - if (!SUCCESS(rc)) { - platform_mutex_destroy(&al->lock); - platform_free(al->heap_id, al->meta_page); - platform_error_log("Failed to create buffer for ref counts\n"); - return STATUS_NO_MEMORY; + + uint64 extent_no = rc_allocator_extent_number(al, extent_addr); + platform_assert(extent_no < al->cfg->extent_capacity); + if (rc_allocator_recovery_extent_is_reserved(al, extent_no)) { + platform_error_log("Cannot rebuild ownership of reserved allocator " + "extent %lu.\n", + extent_no); + return STATUS_BAD_PARAM; } - al->ref_count = platform_buffer_getaddr(&al->bh); - memset(al->ref_count, 0, buffer_size); - // allocate the super block - allocator_alloc(&al->super, &addr, PAGE_TYPE_SUPERBLOCK); - // super block extent should always start from address 0. - platform_assert(addr == RC_ALLOCATOR_BASE_OFFSET); - /* - * Allocate room for the ref counts, use same rounded up size used in buffer - * creation. - */ - rc_extent_count = (buffer_size + al->cfg->io_cfg->extent_size - 1) - / al->cfg->io_cfg->extent_size; - for (uint64 i = 0; i < rc_extent_count; i++) { - allocator_alloc(&al->super, &addr, PAGE_TYPE_SUPERBLOCK); - platform_assert(addr == cfg->io_cfg->extent_size * (i + 1)); + refcount new_refcount = __sync_add_and_fetch(&al->ref_count[extent_no], 1); + if (new_refcount == 0) { + platform_error_log("Allocator recovery refcount overflow for extent " + "%lu.\n", + extent_no); + return STATUS_LIMIT_EXCEEDED; + } else if (new_refcount == AL_NO_REFS) { + new_refcount = __sync_add_and_fetch(&al->ref_count[extent_no], 1); + if (new_refcount == 0) { + platform_error_log("Allocator recovery refcount overflow for extent " + "%lu.\n", + extent_no); + return STATUS_LIMIT_EXCEEDED; + } + platform_assert(type != PAGE_TYPE_INVALID); + rc_allocator_record_allocated_extent(al); + __sync_add_and_fetch(&al->stats.extent_allocs[type], 1); } return STATUS_OK; } void -rc_allocator_deinit(rc_allocator *al) +rc_allocator_recovery_finish(rc_allocator *al) { - platform_buffer_deinit(&al->bh); - al->ref_count = NULL; - platform_mutex_destroy(&al->lock); - platform_free(al->heap_id, al->meta_page); + platform_assert(al != NULL); + platform_assert(!al->map_is_valid); + + /* + * Intentionally no I/O here. A rebuilt map is durable only after a later + * rc_allocator_persist(); another crash before then simply rebuilds it + * again. + */ + al->map_is_valid = TRUE; } + /* *---------------------------------------------------------------------- - * rc_allocator_{mount,unmount} -- + * rc_allocator_load_refcounts -- * - * Loads the file system from disk - * Write the file system to disk + * Populate an attached allocator's refcount map by loading the trusted + * persisted map from its fixed reserved location (the clean-mount path; + * see allocator_load_refcounts()). Allocates the refcount buffer that + * attach (rc_allocator_mount) deliberately left NULL. The caller has + * already confirmed via the superblock that the persisted map is valid; + * the allocator only touches its own map here. *---------------------------------------------------------------------- */ platform_status -rc_allocator_mount(rc_allocator *al, - allocator_config *cfg, - io_handle *io, - platform_heap_id hid, - platform_module_id mid) +rc_allocator_load_refcounts(rc_allocator *al) { - platform_status status; - platform_assert(al != NULL); - ZERO_CONTENTS(al); - al->super.ops = &rc_allocator_ops; - al->cfg = cfg; - al->io = io; - al->heap_id = hid; - - status = platform_mutex_init(&al->lock, mid, al->heap_id); - if (!SUCCESS(status)) { - platform_error_log("Failed to init mutex for the allocator\n"); - return status; - } - status = rc_allocator_init_meta_page(al); + platform_status status = rc_allocator_alloc_refcount_buffer(al); if (!SUCCESS(status)) { - platform_error_log("Failed to init meta page for rc allocator\n"); - platform_mutex_destroy(&al->lock); return status; } - platform_assert(cfg->io_cfg->page_size % 4096 == 0); - platform_assert(cfg->capacity - == cfg->io_cfg->extent_size * cfg->extent_capacity); - platform_assert(cfg->capacity - == cfg->io_cfg->page_size * cfg->page_capacity); - - uint64 buffer_size = cfg->extent_capacity * sizeof(refcount); - buffer_size = ROUNDUP(buffer_size, cfg->io_cfg->page_size); - status = platform_buffer_init(&al->bh, buffer_size); - if (!SUCCESS(status)) { - platform_free(al->heap_id, al->meta_page); - platform_mutex_destroy(&al->lock); - platform_error_log("Failed to create buffer to load ref counts\n"); - return STATUS_NO_MEMORY; - } - al->ref_count = platform_buffer_getaddr(&al->bh); - - // load the meta page from disk. - status = io_read( - io, al->meta_page, al->cfg->io_cfg->page_size, RC_ALLOCATOR_BASE_OFFSET); - if (!SUCCESS(status)) { - platform_free(al->heap_id, al->meta_page); - platform_buffer_deinit(&al->bh); - platform_mutex_destroy(&al->lock); - return status; - } - - status = rc_allocator_validate_disk_geometry(al); - if (!SUCCESS(status)) { - platform_free(al->heap_id, al->meta_page); - platform_buffer_deinit(&al->bh); - platform_mutex_destroy(&al->lock); - return status; - } - - // validate the checksum of the meta page. - checksum128 currChecksum = rc_allocator_meta_page_checksum(al->meta_page); - if (!platform_checksum_is_equal(al->meta_page->checksum, currChecksum)) { - platform_error_log("Corrupt SplinterDB allocator meta page on mount\n"); - platform_free(al->heap_id, al->meta_page); - platform_buffer_deinit(&al->bh); - platform_mutex_destroy(&al->lock); - return STATUS_BAD_PARAM; - } - - // load the ref counts from disk. - status = io_read(io, al->ref_count, buffer_size, cfg->io_cfg->extent_size); + // Load the trusted refcount map from its fixed reserved location. + uint64 buffer_size = rc_allocator_refcount_buffer_size(al->cfg); + status = + io_read(al->io, + al->ref_count, + buffer_size, + RC_ALLOCATOR_REFCOUNT_MAP_EXTENT * al->cfg->io_cfg->extent_size); if (!SUCCESS(status)) { - platform_free(al->heap_id, al->meta_page); - platform_buffer_deinit(&al->bh); - platform_mutex_destroy(&al->lock); return status; } + // Compute curr_allocated authoritatively from the loaded map (set, not + // accumulate), so this is correct even if the allocator's stats were not + // freshly zeroed. + al->stats.curr_allocated = 0; for (uint64 i = 0; i < al->cfg->extent_capacity; i++) { if (al->ref_count[i] != 0) { al->stats.curr_allocated++; } } + al->map_is_valid = TRUE; return STATUS_OK; } - -void -rc_allocator_unmount(rc_allocator *al) +/* + *---------------------------------------------------------------------- + * rc_allocator_persist -- + * + * Write the refcount map to its fixed reserved location and make it + * durable. Returns the base address of the persisted map in *state_addr, + * which the caller records as the superblock's allocation_state_addr only + * after this returns -- so the "map is trustworthy" flag never becomes + * durable before the map itself. Does not tear down the allocator; the + * caller invokes rc_allocator_deinit() separately. + *---------------------------------------------------------------------- + */ +platform_status +rc_allocator_persist(rc_allocator *al, uint64 *state_addr) { - platform_status status; + platform_assert(al->map_is_valid); - // persist the ref counts upon unmount. - uint64 buffer_size = al->cfg->extent_capacity * sizeof(refcount); + uint64 map_addr = + RC_ALLOCATOR_REFCOUNT_MAP_EXTENT * al->cfg->io_cfg->extent_size; + uint64 buffer_size = rc_allocator_refcount_buffer_size(al->cfg); uint32 io_size = ROUNDUP(buffer_size, al->cfg->io_cfg->page_size); - status = - io_write(al->io, al->ref_count, io_size, al->cfg->io_cfg->extent_size); - platform_assert_status_ok(status); - rc_allocator_deinit(al); + + platform_status status = io_write(al->io, al->ref_count, io_size, map_addr); + if (!SUCCESS(status)) { + return status; + } + status = io_durable_barrier(al->io); + if (!SUCCESS(status)) { + return status; + } + if (state_addr != NULL) { + *state_addr = map_addr; + } + return STATUS_OK; } @@ -639,97 +459,17 @@ rc_allocator_get_ref(rc_allocator *al, uint64 addr) } -/* - *---------------------------------------------------------------------- - * rc_allocator_get_[capacity,super_addr] -- - * - * returns the struct/parameter - *---------------------------------------------------------------------- - */ -uint64 -rc_allocator_get_capacity(rc_allocator *al) -{ - return al->cfg->capacity; -} - -platform_status -rc_allocator_get_super_addr(rc_allocator *al, - allocator_root_id allocator_root_id, - uint64 *addr) -{ - platform_status status = STATUS_NOT_FOUND; - - platform_mutex_lock(&al->lock); - for (uint8 idx = 0; idx < RC_ALLOCATOR_MAX_ROOT_IDS; idx++) { - if (al->meta_page->splinters[idx] == allocator_root_id) { - // have already seen this table before, return existing addr. - *addr = (1 + idx) * al->cfg->io_cfg->page_size; - status = STATUS_OK; - break; - } - } - - platform_mutex_unlock(&al->lock); - return status; -} - -platform_status -rc_allocator_alloc_super_addr(rc_allocator *al, - allocator_root_id allocator_root_id, - uint64 *addr) -{ - platform_status status = STATUS_NOT_FOUND; - - platform_mutex_lock(&al->lock); - for (uint8 idx = 0; idx < RC_ALLOCATOR_MAX_ROOT_IDS; idx++) { - if (al->meta_page->splinters[idx] == INVALID_ALLOCATOR_ROOT_ID) { - // assign the first available slot and update the on disk metadata. - al->meta_page->splinters[idx] = allocator_root_id; - *addr = (1 + idx) * al->cfg->io_cfg->page_size; - al->meta_page->checksum = - rc_allocator_meta_page_checksum(al->meta_page); - platform_status io_status = io_write(al->io, - al->meta_page, - al->cfg->io_cfg->page_size, - RC_ALLOCATOR_BASE_OFFSET); - platform_assert_status_ok(io_status); - status = STATUS_OK; - break; - } - } - - platform_mutex_unlock(&al->lock); - return status; -} - -void -rc_allocator_remove_super_addr(rc_allocator *al, - allocator_root_id allocator_root_id) -{ - platform_mutex_lock(&al->lock); - - for (uint8 idx = 0; idx < RC_ALLOCATOR_MAX_ROOT_IDS; idx++) { - /* - * clear out the mapping for this splinter table and update on disk - * metadata. - */ - if (al->meta_page->splinters[idx] == allocator_root_id) { - al->meta_page->splinters[idx] = INVALID_ALLOCATOR_ROOT_ID; - al->meta_page->checksum = - rc_allocator_meta_page_checksum(al->meta_page); - platform_status status = io_write(al->io, - al->meta_page, - al->cfg->io_cfg->page_size, - RC_ALLOCATOR_BASE_OFFSET); - platform_assert_status_ok(status); - platform_mutex_unlock(&al->lock); - return; - } - } - - platform_mutex_unlock(&al->lock); - // Couldn't find the splinter id in the meta page. - platform_assert(0, "Couldn't find existing splinter table in meta page"); +/* + *---------------------------------------------------------------------- + * rc_allocator_get_[capacity,super_addr] -- + * + * returns the struct/parameter + *---------------------------------------------------------------------- + */ +uint64 +rc_allocator_get_capacity(rc_allocator *al) +{ + return al->cfg->capacity; } uint64 @@ -823,34 +563,6 @@ rc_allocator_in_use(rc_allocator *al) return al->stats.curr_allocated; } -/* - *---------------------------------------------------------------------- - * rc_allocator_assert_noleaks -- - * - * Asserts that the allocations of each type are completely matched by - * deallocations by operations that do something like create / destroy - * of objects. Primitive function to do some basic cross-checking of - * these operations. - *---------------------------------------------------------------------- - */ -void -rc_allocator_assert_noleaks(rc_allocator *al) -{ - for (page_type type = PAGE_TYPE_FIRST; type < NUM_PAGE_TYPES; type++) { - // Log pages and super-block page are never deallocated. - if ((type == PAGE_TYPE_LOG) || (type == PAGE_TYPE_SUPERBLOCK)) { - continue; - } - if (al->stats.extent_allocs[type] != al->stats.extent_deallocs[type]) { - platform_default_log("assert_noleaks: leak found\n"); - platform_default_log("\n"); - rc_allocator_print_stats(al); - rc_allocator_print_allocated(al); - platform_assert(0); - } - } -} - /* *---------------------------------------------------------------------- * rc_allocator_print_stats() -- @@ -951,3 +663,268 @@ rc_allocator_print_allocated(rc_allocator *al) nextents_found, size_str(nextents_found * extent_size)); } + +/* + *---------------------------------------------------------------------- + * rc_allocator_assert_noleaks -- + * + * Asserts that the allocations of each type are completely matched by + * deallocations by operations that do something like create / destroy + * of objects. Primitive function to do some basic cross-checking of + * these operations. + *---------------------------------------------------------------------- + */ +void +rc_allocator_assert_noleaks(rc_allocator *al) +{ + for (page_type type = PAGE_TYPE_FIRST; type < NUM_PAGE_TYPES; type++) { + // Log pages and super-block page are never deallocated. + if ((type == PAGE_TYPE_LOG) || (type == PAGE_TYPE_SUPERBLOCK)) { + continue; + } + if (al->stats.extent_allocs[type] != al->stats.extent_deallocs[type]) { + platform_default_log("assert_noleaks: leak found\n"); + platform_default_log("\n"); + rc_allocator_print_stats(al); + rc_allocator_print_allocated(al); + platform_assert(0); + } + } +} + +// allocator.h functions + +allocator_config * +rc_allocator_get_config_virtual(allocator *a) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_get_config(al); +} + +platform_status +rc_allocator_alloc_virtual(allocator *a, uint64 *addr, page_type type) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_alloc(al, addr, type); +} + +refcount +rc_allocator_inc_ref_virtual(allocator *a, uint64 addr) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_inc_ref(al, addr); +} + +refcount +rc_allocator_dec_ref_virtual(allocator *a, uint64 addr, page_type type) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_dec_ref(al, addr, type); +} + +refcount +rc_allocator_get_ref_virtual(allocator *a, uint64 addr) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_get_ref(al, addr); +} + +platform_status +rc_allocator_recovery_begin_virtual(allocator *a) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_recovery_begin(al); +} + +platform_status +rc_allocator_recovery_record_reference_virtual(allocator *a, + uint64 addr, + page_type type) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_recovery_record_reference(al, addr, type); +} + +void +rc_allocator_recovery_finish_virtual(allocator *a) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_recovery_finish(al); +} + + +platform_status +rc_allocator_load_refcounts_virtual(allocator *a) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_load_refcounts(al); +} + +platform_status +rc_allocator_persist_virtual(allocator *a, uint64 *state_addr) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_persist(al, state_addr); +} + + +uint64 +rc_allocator_in_use_virtual(allocator *a) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_in_use(al); +} + +uint64 +rc_allocator_get_capacity_virtual(allocator *a) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_get_capacity(al); +} + +void +rc_allocator_assert_noleaks_virtual(allocator *a) +{ + rc_allocator *al = (rc_allocator *)a; + rc_allocator_assert_noleaks(al); +} + +void +rc_allocator_print_stats_virtual(allocator *a) +{ + rc_allocator *al = (rc_allocator *)a; + rc_allocator_print_stats(al); +} + +void +rc_allocator_print_allocated_virtual(allocator *a) +{ + rc_allocator *al = (rc_allocator *)a; + rc_allocator_print_allocated(al); +} + +const static allocator_ops rc_allocator_ops = { + .get_config = rc_allocator_get_config_virtual, + .alloc = rc_allocator_alloc_virtual, + .inc_ref = rc_allocator_inc_ref_virtual, + .dec_ref = rc_allocator_dec_ref_virtual, + .get_ref = rc_allocator_get_ref_virtual, + .recovery_begin = rc_allocator_recovery_begin_virtual, + .recovery_record_reference = rc_allocator_recovery_record_reference_virtual, + .recovery_finish = rc_allocator_recovery_finish_virtual, + .load_refcounts = rc_allocator_load_refcounts_virtual, + .persist = rc_allocator_persist_virtual, + .in_use = rc_allocator_in_use_virtual, + .get_capacity = rc_allocator_get_capacity_virtual, + .assert_noleaks = rc_allocator_assert_noleaks_virtual, + .print_stats = rc_allocator_print_stats_virtual, + .print_allocated = rc_allocator_print_allocated_virtual, +}; + +/* + *---------------------------------------------------------------------- + * rc_allocator_[de]init -- + * + * [de]initialize an allocator + *---------------------------------------------------------------------- + */ +platform_status +rc_allocator_init(rc_allocator *al, + allocator_config *cfg, + io_handle *io, + platform_heap_id hid, + platform_module_id mid) +{ + platform_status rc; + platform_assert(al != NULL); + ZERO_CONTENTS(al); + al->super.ops = &rc_allocator_ops; + al->cfg = cfg; + al->io = io; + al->heap_id = hid; + + rc = rc_allocator_valid_config(cfg); + if (!SUCCESS(rc)) { + return rc; + } + + rc = platform_mutex_init(&al->lock, mid, al->heap_id); + if (!SUCCESS(rc)) { + platform_error_log("Failed to init mutex for the allocator\n"); + return rc; + } + + rc = rc_allocator_init_refcounts(al); + if (!SUCCESS(rc)) { + platform_mutex_destroy(&al->lock); + return rc; + } + + /* + * A fresh map is synthesized correctly in memory -- there is nothing to + * load, so it's trustworthy immediately, like a map that just finished a + * rebuild. It is not persisted here, though: the superblock that mkfs + * writes records allocation_state_addr == 0, so a crash before a later + * clean close enters rebuild recovery rather than trusting this in-memory + * map. The map is persisted only by rc_allocator_persist(). + */ + al->map_is_valid = TRUE; + return STATUS_OK; +} + +void +rc_allocator_deinit(rc_allocator *al) +{ + if (al->ref_count) { + platform_buffer_deinit(&al->bh); + al->ref_count = NULL; + } + platform_mutex_destroy(&al->lock); +} + +/* + *---------------------------------------------------------------------- + * rc_allocator_mount -- + * + * Attach an allocator to an existing device: initialize the in-memory + * structures only. The refcount buffer is deliberately left NULL so that + * using the allocator before its map is populated faults immediately; the + * caller MUST next call rc_allocator_load_refcounts() (clean mount) or + * rc_allocator_recovery_begin() (rebuild), each of which allocates and + * populates the buffer. A rebuild therefore never reads a map it would + * discard. + *---------------------------------------------------------------------- + */ +platform_status +rc_allocator_mount(rc_allocator *al, + allocator_config *cfg, + io_handle *io, + platform_heap_id hid, + platform_module_id mid) +{ + platform_status status; + + platform_assert(al != NULL); + ZERO_CONTENTS(al); + al->super.ops = &rc_allocator_ops; + al->cfg = cfg; + al->io = io; + al->heap_id = hid; + + status = platform_mutex_init(&al->lock, mid, al->heap_id); + if (!SUCCESS(status)) { + platform_error_log("Failed to init mutex for the allocator\n"); + return status; + } + + platform_assert(cfg->io_cfg->page_size % 4096 == 0); + platform_assert(cfg->capacity + == cfg->io_cfg->extent_size * cfg->extent_capacity); + platform_assert(cfg->capacity + == cfg->io_cfg->page_size * cfg->page_capacity); + + // Attach only: ref_count is left NULL (and map_is_valid FALSE) until + // load_refcounts / recovery_begin populates the map, so a premature use of + // the allocator faults at once. + return STATUS_OK; +} diff --git a/src/rc_allocator.h b/src/rc_allocator.h index 06ed7256..91eccb72 100644 --- a/src/rc_allocator.h +++ b/src/rc_allocator.h @@ -15,34 +15,6 @@ #include "allocator.h" #include "util.h" -/* - * In the current system, every Splinter instance has a superblock, one - * for each table that is mapped to the Splinter instance. This limit - * is the max number of superblocks (special pages) that can be accessed. - * All of these superblocks are required to be on the 1st extent. - */ -#define RC_ALLOCATOR_MAX_ROOT_IDS (30) - -/* - *---------------------------------------------------------------------- - * rc_allocator_meta_page -- Disk-resident structure. - * - * An on disk structure to hold the bootstrap disk geometry and the super block - * addresses for all Splinter tables using this allocator. The geometry lives at - * offset 0 so open can read it before mounting the rest of SplinterDB. - *---------------------------------------------------------------------- - */ -typedef struct ONDISK rc_allocator_meta_page { - disk_geometry geometry; - allocator_root_id splinters[RC_ALLOCATOR_MAX_ROOT_IDS]; - checksum128 checksum; -} rc_allocator_meta_page; - -_Static_assert(offsetof(rc_allocator_meta_page, geometry) == 0, - "disk geometry should be first field in meta_page struct"); -_Static_assert(sizeof(rc_allocator_meta_page) <= IO_DEFAULT_PAGE_SIZE, - "allocator meta page must fit in the default page size"); - /* *---------------------------------------------------------------------- * rc_allocator_stats -- @@ -61,20 +33,26 @@ typedef struct rc_allocator_stats { *---------------------------------------------------------------------- */ typedef struct rc_allocator { - allocator super; - allocator_config *cfg; - buffer_handle bh; - refcount *ref_count; - uint64 hand; - io_handle *io; - rc_allocator_meta_page *meta_page; + allocator super; + allocator_config *cfg; + buffer_handle bh; + refcount *ref_count; + uint64 hand; + io_handle *io; + + /* Serializes refcount-map mutations that must be atomic (e.g. stats). */ + platform_mutex lock; + platform_heap_id heap_id; /* - * mutex to synchronize updates to super block addresses of the splinter - * tables in the meta page. + * True once the refcount map is trustworthy: set by a clean + * rc_allocator_load_refcounts(al, rebuild=FALSE) load, or by + * rc_allocator_rebuild_finish() after a rebuild completes. False from + * rc_allocator_mount() (attach) until then, including throughout an + * in-progress rebuild. rc_allocator_persist() asserts this is true: an + * incomplete rebuilt map must never be written back to disk. */ - platform_mutex lock; - platform_heap_id heap_id; + bool32 map_is_valid; // Stats -- not distributed for now rc_allocator_stats stats; @@ -90,19 +68,14 @@ rc_allocator_init(rc_allocator *al, void rc_allocator_deinit(rc_allocator *al); +/* + * Attach to an existing device but do not read the persisted map. The caller + * then populates the map exactly once via allocator_load_refcounts() or + * allocator_recovery_begin/finish. + */ platform_status rc_allocator_mount(rc_allocator *al, allocator_config *cfg, io_handle *io, platform_heap_id hid, platform_module_id mid); - -platform_status -rc_allocator_read_disk_geometry(const char *filename, disk_geometry *geometry); - -platform_status -rc_allocator_disk_geometry_matches_config(const disk_geometry *geometry, - const allocator_config *cfg); - -void -rc_allocator_unmount(rc_allocator *al); diff --git a/src/shard_log.c b/src/shard_log.c index c0eee131..3f6934d9 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -25,40 +25,6 @@ static uint64 shard_log_magic_idx = 0; -int -shard_log_write(log_handle *log, key tuple_key, message msg, uint64 generation); -uint64 -shard_log_addr(log_handle *log); -uint64 -shard_log_meta_addr(log_handle *log); -uint64 -shard_log_magic(log_handle *log); - -static log_ops shard_log_ops = { - .write = shard_log_write, - .addr = shard_log_addr, - .meta_addr = shard_log_meta_addr, - .magic = shard_log_magic, -}; - -void -shard_log_iterator_curr(iterator *itor, key *curr_key, message *msg); -bool32 -shard_log_iterator_can_prev(iterator *itor); -bool32 -shard_log_iterator_can_next(iterator *itor); -platform_status -shard_log_iterator_next(iterator *itor); - - -const static iterator_ops shard_log_iterator_ops = { - .curr = shard_log_iterator_curr, - .can_prev = shard_log_iterator_can_prev, - .can_next = shard_log_iterator_can_next, - .next = shard_log_iterator_next, - .print = NULL, -}; - static const page_type shard_log_page_type_table[NUM_BLOB_BATCHES + 1] = { PAGE_TYPE_LOG, [1 ... NUM_BLOB_BATCHES] = PAGE_TYPE_BLOB, @@ -76,6 +42,12 @@ shard_log_pages_per_extent(shard_log_config *cfg) return cache_config_pages_per_extent(cfg->cache_cfg); } +static inline uint64 +shard_log_extent_size(shard_log_config *cfg) +{ + return cache_config_extent_size(cfg->cache_cfg); +} + static inline checksum128 shard_log_checksum(shard_log_config *cfg, page_handle *page) { @@ -93,56 +65,10 @@ page_handle * shard_log_alloc(shard_log *log, uint64 *next_extent) { uint64 addr = mini_alloc_page(&log->mini, 0, next_extent); - return cache_alloc(log->cc, addr, PAGE_TYPE_LOG); -} - -platform_status -shard_log_init(shard_log *log, cache *cc, shard_log_config *cfg) -{ - memset(log, 0, sizeof(shard_log)); - log->cc = cc; - log->cfg = cfg; - log->super.ops = &shard_log_ops; - - uint64 magic_idx = __sync_fetch_and_add(&shard_log_magic_idx, 1); - log->magic = platform_checksum64(&magic_idx, sizeof(uint64), cfg->seed); - - allocator *al = cache_get_allocator(cc); - platform_status rc = allocator_alloc(al, &log->meta_head, PAGE_TYPE_LOG); - platform_assert_status_ok(rc); - - for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { - shard_log_thread_data *thread_data = - shard_log_get_thread_data(log, thr_i); - thread_data->addr = SHARD_UNMAPPED; - thread_data->offset = 0; - } - - log->addr = mini_init_with_types(&log->mini, - cc, - log->meta_head, - 0, - NUM_BLOB_BATCHES + 1, - PAGE_TYPE_LOG, - shard_log_page_type_table); - // platform_default_log("addr: %lu meta_head: %lu\n", log->addr, - // log->meta_head); - - return STATUS_OK; -} - -void -shard_log_zap(shard_log *log) -{ - cache *cc = log->cc; - - for (threadid i = 0; i < MAX_THREADS; i++) { - shard_log_thread_data *thread_data = shard_log_get_thread_data(log, i); - thread_data->addr = SHARD_UNMAPPED; - thread_data->offset = 0; + if (addr == 0) { + return NULL; } - - mini_dec_ref(cc, log->meta_head, PAGE_TYPE_LOG); + return cache_alloc(log->cc, addr, PAGE_TYPE_LOG); } /* @@ -152,11 +78,12 @@ shard_log_zap(shard_log *log) * ------------------------------------------------------------------------- */ struct ONDISK log_entry { - uint64 generation; + uint64 memtable_generation; + uint64 leaf_generation; ondisk_tuple tuple; }; -#define INVALID_GENERATION ((uint64) - 1) +#define INVALID_LOG_GENERATION ((uint64) - 1) static key log_entry_key(log_entry *le) @@ -200,7 +127,19 @@ static bool32 terminal_log_entry(shard_log_config *cfg, char *page, log_entry *le) { return page + shard_log_page_size(cfg) - (char *)le < sizeof(log_entry) - || le->generation == INVALID_GENERATION; + || le->memtable_generation == INVALID_LOG_GENERATION; +} + +static inline void +log_entry_set_terminal(log_entry *le) +{ + /* + * terminal_log_entry() tests memtable_generation. Mark both generation + * fields invalid so the terminator cannot be confused with a record by + * diagnostics or a future format validator. + */ + le->leaf_generation = INVALID_LOG_GENERATION; + le->memtable_generation = INVALID_LOG_GENERATION; } static log_entry * @@ -216,7 +155,10 @@ get_new_page_for_thread(shard_log *log, { uint64 next_extent; - *page = shard_log_alloc(log, &next_extent); + *page = shard_log_alloc(log, &next_extent); + if (*page == NULL) { + return -1; + } thread_data->addr = (*page)->disk_addr; shard_log_hdr *hdr = (shard_log_hdr *)(*page)->data; hdr->magic = log->magic; @@ -227,9 +169,15 @@ get_new_page_for_thread(shard_log *log, } int -shard_log_write(log_handle *logh, key tuple_key, message msg, uint64 generation) +shard_log_write(log_handle *logh, + key tuple_key, + message msg, + uint64 memtable_generation, + uint64 leaf_generation) { debug_assert(key_is_user_key(tuple_key)); + debug_assert(memtable_generation != INVALID_LOG_GENERATION); + debug_assert(leaf_generation != INVALID_LOG_GENERATION); shard_log *log = (shard_log *)logh; cache *cc = log->cc; @@ -290,13 +238,13 @@ shard_log_write(log_handle *logh, key tuple_key, message msg, uint64 generation) if (free_space < new_entry_size) { if (sizeof(log_entry) <= free_space) { - cursor->generation = INVALID_GENERATION; + log_entry_set_terminal(cursor); } hdr->checksum = shard_log_checksum(log->cfg, page); cache_unlock(cc, page); cache_unclaim(cc, page); - cache_page_sync(cc, page, FALSE, PAGE_TYPE_LOG); + cache_page_writeback(cc, page, FALSE, PAGE_TYPE_LOG); cache_unget(cc, page); if (get_new_page_for_thread(log, thread_data, &page)) { @@ -309,7 +257,8 @@ shard_log_write(log_handle *logh, key tuple_key, message msg, uint64 generation) hdr = (shard_log_hdr *)page->data; } - cursor->generation = generation; + cursor->memtable_generation = memtable_generation; + cursor->leaf_generation = leaf_generation; copy_tuple_to_ondisk_tuple(&cursor->tuple, tuple_key, msg); hdr->num_entries++; @@ -332,25 +281,104 @@ shard_log_write(log_handle *logh, key tuple_key, message msg, uint64 generation) return 0; } -uint64 -shard_log_addr(log_handle *logh) +/* + * shard_log_seal -- + * + * Finalize and retire a log stream, terminally. Finalizes every currently + * active per-thread append page (bounded by MAX_THREADS; it does not walk + * the historical log): a terminal record (where there is room) and checksum + * make each page readable by shard_log_iterator_init(). Then it releases + * the mini-allocator's unused reserve and frees the handle. After seal the + * handle is invalid; the caller retains the identity it captured earlier + * (shard_log_get_head(), fixed at creation) to reopen the stream + * for replay and, eventually, to free its extents via log_dec_ref(). + * + * The caller must prevent concurrent shard_log_write() and seal calls. + * seal itself issues no writeback or durable barrier: to make the sealed + * pages durable, the caller takes cache_writeback_dirty() followed by a + * durable barrier. + */ +platform_status +shard_log_seal(log_handle *logh) { shard_log *log = (shard_log *)logh; - return log->addr; + cache *cc = log->cc; + + for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { + shard_log_thread_data *thread_data = + shard_log_get_thread_data(log, thr_i); + uint64 addr = thread_data->addr; + if (addr == SHARD_UNMAPPED) { + continue; + } + + page_handle *page = cache_get(cc, addr, TRUE, PAGE_TYPE_LOG); + uint64 wait = 1; + while (!cache_try_claim(cc, page)) { + /* + * Even though the stream is quiescent (no concurrent writers/seals), + * the background cache evictor can transiently hold the claim on a + * cleaned log page before it drains our read-ref, so we still retry + * rather than assert. + */ + cache_unget(cc, page); + platform_sleep_ns(wait); + wait = wait > 1024 ? wait : 2 * wait; + page = cache_get(cc, addr, TRUE, PAGE_TYPE_LOG); + } + cache_lock(cc, page); + + debug_assert(thread_data->addr == addr); + debug_assert(thread_data->offset >= sizeof(shard_log_hdr)); + debug_assert(thread_data->offset <= shard_log_page_size(log->cfg)); + + shard_log_hdr *hdr = (shard_log_hdr *)page->data; + log_entry *cursor = (log_entry *)(page->data + thread_data->offset); + uint64 free_space = shard_log_page_size(log->cfg) - thread_data->offset; + if (sizeof(log_entry) <= free_space) { + log_entry_set_terminal(cursor); + } + hdr->checksum = shard_log_checksum(log->cfg, page); + + cache_unlock(cc, page); + cache_unclaim(cc, page); + cache_unget(cc, page); + + /* Subsequent writes must allocate a new append page. */ + thread_data->addr = SHARD_UNMAPPED; + thread_data->offset = 0; + } + + /* + * The stream is now immutable. Release the mini-allocator's unused + * per-batch reserve so no future allocation touches this stream, and free + * the handle. The caller already holds the stream's identity (captured at + * creation) and later frees the on-disk extents via log_dec_ref(). + */ + mini_release(&log->mini); + platform_free(log->heap_id, log); + return STATUS_OK; } -uint64 -shard_log_meta_addr(log_handle *logh) +void +log_dec_ref(cache *cc, const log_head *segment) { - shard_log *log = (shard_log *)logh; - return log->meta_head; + if (segment->meta_addr == 0) { + return; + } + refcount ref = mini_dec_ref(cc, segment->meta_addr, PAGE_TYPE_LOG); + platform_assert(ref == 0); } -uint64 -shard_log_magic(log_handle *logh) +log_head +shard_log_get_head(log_handle *logh) { shard_log *log = (shard_log *)logh; - return log->magic; + return (log_head){ + .addr = log->addr, + .meta_addr = log->meta_head, + .magic = log->magic, + }; } bool32 @@ -369,34 +397,240 @@ shard_log_next_extent_addr(shard_log_config *cfg, page_handle *page) return hdr->next_extent_addr; } -int -shard_log_compare(const void *p1, const void *p2, void *unused) +/* + * Bytes appended to the stream so far. The mini-allocator already tracks the + * extents it has handed out across all of the stream's batches (data and blob), + * counting each as it is reserved -- the same measure memtable_is_full() uses + * for a memtable. Subtracting the fixed overhead recorded at init means a + * fresh stream reports 0, so a caller comparing against a threshold cannot be + * tricked into rotating a stream that has had nothing written to it. + */ +uint64 +shard_log_get_size(log_handle *logh) { - log_entry **le1 = (log_entry **)p1; - log_entry **le2 = (log_entry **)p2; - return (*le1)->generation - (*le2)->generation; + shard_log *log = (shard_log *)logh; + return (mini_num_extents(&log->mini) - log->initial_extents) + * shard_log_extent_size(log->cfg); +} + +static log_ops shard_log_ops = { + .write = shard_log_write, + .seal = shard_log_seal, + .head = shard_log_get_head, + .size = shard_log_get_size, +}; + +static platform_status +shard_log_init(shard_log *log, cache *cc, shard_log_config *cfg) +{ + memset(log, 0, sizeof(shard_log)); + log->cc = cc; + log->cfg = cfg; + log->super.ops = &shard_log_ops; + + uint64 magic_idx = __sync_fetch_and_add(&shard_log_magic_idx, 1); + log->magic = platform_checksum64(&magic_idx, sizeof(uint64), cfg->seed); + + allocator *al = cache_get_allocator(cc); + platform_status rc = allocator_alloc(al, &log->meta_head, PAGE_TYPE_LOG); + platform_assert_status_ok(rc); + + for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { + shard_log_thread_data *thread_data = + shard_log_get_thread_data(log, thr_i); + thread_data->addr = SHARD_UNMAPPED; + thread_data->offset = 0; + } + + log->addr = mini_init_with_types(&log->mini, + cc, + log->meta_head, + 0, + NUM_BLOB_BATCHES + 1, + PAGE_TYPE_LOG, + shard_log_page_type_table); + // platform_default_log("addr: %lu meta_head: %lu\n", log->addr, + // log->meta_head); + + // Baseline for shard_log_get_size(): the stream's fixed overhead. + log->initial_extents = mini_num_extents(&log->mini); + + return STATUS_OK; } log_handle * -log_create(cache *cc, log_config *lcfg, platform_heap_id hid) +shard_log_create(cache *cc, shard_log_config *cfg, platform_heap_id hid) { - shard_log_config *cfg = (shard_log_config *)lcfg; - shard_log *slog = TYPED_MALLOC(hid, slog); + shard_log *slog = TYPED_MALLOC(hid, slog); if (slog == NULL) { - platform_error_log("log_create: failed to allocate shard_log\n"); + platform_error_log("shard_log_create: failed to allocate shard_log\n"); return NULL; } platform_status rc = shard_log_init(slog, cc, cfg); if (!SUCCESS(rc)) { - platform_error_log("log_create: shard_log_init failed: %s\n", + platform_error_log("shard_log_create: shard_log_init failed: %s\n", platform_status_to_string(rc)); platform_free(hid, slog); return NULL; } + // Remember the heap so log_seal() can free the handle without the caller + // touching platform_free() directly. + slog->heap_id = hid; return (log_handle *)slog; } +int +shard_log_compare(const void *p1, const void *p2, void *unused) +{ + log_entry **le1 = (log_entry **)p1; + log_entry **le2 = (log_entry **)p2; + + if ((*le1)->memtable_generation < (*le2)->memtable_generation) { + return -1; + } + if ((*le1)->memtable_generation > (*le2)->memtable_generation) { + return 1; + } + if ((*le1)->leaf_generation < (*le2)->leaf_generation) { + return -1; + } + if ((*le1)->leaf_generation > (*le2)->leaf_generation) { + return 1; + } + return 0; +} + +void +shard_log_iterator_curr(iterator *itorh, key *curr_key, message *msg) +{ + shard_log_iterator *itor = (shard_log_iterator *)itorh; + *curr_key = log_entry_key(itor->entries[itor->pos]); + *msg = log_entry_message(itor->cc, itor->entries[itor->pos]); +} + +static void +shard_log_iterator_curr_generations(log_iterator *itorh, + uint64 *memtable_generation, + uint64 *leaf_generation) +{ + shard_log_iterator *itor = (shard_log_iterator *)itorh; + platform_assert(itor->pos < itor->num_entries); + *memtable_generation = itor->entries[itor->pos]->memtable_generation; + *leaf_generation = itor->entries[itor->pos]->leaf_generation; +} + +bool32 +shard_log_iterator_can_prev(iterator *itorh) +{ + shard_log_iterator *itor = (shard_log_iterator *)itorh; + return itor->pos >= 0; +} + +bool32 +shard_log_iterator_can_next(iterator *itorh) +{ + shard_log_iterator *itor = (shard_log_iterator *)itorh; + return itor->pos < itor->num_entries; +} + platform_status +shard_log_iterator_next(iterator *itorh) +{ + shard_log_iterator *itor = (shard_log_iterator *)itorh; + itor->pos++; + return STATUS_OK; +} + +/* + *----------------------------------------------------------------------------- + * shard_log_config_init -- + * + * Initialize shard_log config values + *----------------------------------------------------------------------------- + */ +void +shard_log_config_init(shard_log_config *log_cfg, + cache_config *cache_cfg, + data_config *data_cfg) +{ + ZERO_CONTENTS(log_cfg); + log_cfg->cache_cfg = cache_cfg; + log_cfg->data_cfg = data_cfg; + log_cfg->seed = HASH_SEED; + log_cfg->blob_cfg = (blob_build_config){ + .extent_batch = 1, + .page_batch = 2, + .subpage_batch = 3, + .alignment = cache_config_page_size(cache_cfg), + }; +} + +void +shard_log_print(shard_log *log) +{ + cache *cc = log->cc; + uint64 extent_addr = log->addr; + shard_log_config *cfg = log->cfg; + uint64 magic = log->magic; + data_config *dcfg = cfg->data_cfg; + uint64 pages_per_extent = shard_log_pages_per_extent(cfg); + allocator *al = cache_get_allocator(cc); + + while (extent_addr != 0 && allocator_get_refcount(al, extent_addr) > 0) { + cache_prefetch(cc, extent_addr, PAGE_TYPE_LOG); + uint64 next_extent_addr = 0; + for (uint64 i = 0; i < pages_per_extent; i++) { + uint64 page_addr = extent_addr + i * shard_log_page_size(cfg); + page_handle *page = cache_get(cc, page_addr, TRUE, PAGE_TYPE_LOG); + if (shard_log_valid(cfg, page, magic)) { + next_extent_addr = shard_log_next_extent_addr(cfg, page); + for (log_entry *le = first_log_entry(page->data); + !terminal_log_entry(cfg, page->data, le); + le = log_entry_next(le)) + { + platform_default_log( + "%s -- %s%s : memtable=%lu leaf=%lu\n", + key_string(dcfg, log_entry_key(le)), + log_entry_message_is_blob(le) ? "(blob) " : "", + message_string(dcfg, log_entry_message(cc, le)), + le->memtable_generation, + le->leaf_generation); + } + } + cache_unget(cc, page); + } + extent_addr = next_extent_addr; + } +} + +static void +shard_log_iterator_deinit(log_iterator *itorh) +{ + shard_log_iterator *itor = (shard_log_iterator *)itorh; + platform_heap_id hid = itor->heap_id; + if (itor->contents != NULL) { + platform_free(hid, itor->contents); + } + if (itor->entries != NULL) { + platform_free(hid, itor->entries); + } + platform_free(hid, itor); // the handle, from shard_log_iterator_create() +} + +const static iterator_ops shard_log_iterator_ops = { + .curr = shard_log_iterator_curr, + .can_prev = shard_log_iterator_can_prev, + .can_next = shard_log_iterator_can_next, + .next = shard_log_iterator_next, + .print = NULL, +}; + +const static log_iterator_ops shard_log_log_iterator_ops = { + .curr_generations = shard_log_iterator_curr_generations, + .deinit = shard_log_iterator_deinit, +}; + +static platform_status shard_log_iterator_init(cache *cc, shard_log_config *cfg, platform_heap_id hid, @@ -414,10 +648,12 @@ shard_log_iterator_init(cache *cc, uint64 contents_size; memset(itor, 0, sizeof(shard_log_iterator)); - itor->super.ops = &shard_log_iterator_ops; - itor->cc = cc; - itor->cfg = cfg; - allocator *al = cache_get_allocator(cc); + itor->super.super.ops = &shard_log_iterator_ops; // generic iterator + itor->super.ops = &shard_log_log_iterator_ops; // log_iterator + itor->heap_id = hid; + itor->cc = cc; + itor->cfg = cfg; + allocator *al = cache_get_allocator(cc); // traverse the log extents and calculate the required space extent_addr = addr; @@ -508,104 +744,26 @@ shard_log_iterator_init(cache *cc, return STATUS_OK; } -void -shard_log_iterator_deinit(platform_heap_id hid, shard_log_iterator *itor) +log_iterator * +shard_log_iterator_create(cache *cc, + shard_log_config *cfg, + platform_heap_id hid, + log_head head) { - if (itor->contents != NULL) { - platform_free(hid, itor->contents); - } - if (itor->entries != NULL) { - platform_free(hid, itor->entries); + shard_log_iterator *itor = TYPED_MALLOC(hid, itor); + if (itor == NULL) { + platform_error_log("shard_log_iterator_create: failed to allocate " + "shard_log_iterator\n"); + return NULL; } -} - -void -shard_log_iterator_curr(iterator *itorh, key *curr_key, message *msg) -{ - shard_log_iterator *itor = (shard_log_iterator *)itorh; - *curr_key = log_entry_key(itor->entries[itor->pos]); - *msg = log_entry_message(itor->cc, itor->entries[itor->pos]); -} - -bool32 -shard_log_iterator_can_prev(iterator *itorh) -{ - shard_log_iterator *itor = (shard_log_iterator *)itorh; - return itor->pos >= 0; -} - -bool32 -shard_log_iterator_can_next(iterator *itorh) -{ - shard_log_iterator *itor = (shard_log_iterator *)itorh; - return itor->pos < itor->num_entries; -} - -platform_status -shard_log_iterator_next(iterator *itorh) -{ - shard_log_iterator *itor = (shard_log_iterator *)itorh; - itor->pos++; - return STATUS_OK; -} - -/* - *----------------------------------------------------------------------------- - * shard_log_config_init -- - * - * Initialize shard_log config values - *----------------------------------------------------------------------------- - */ -void -shard_log_config_init(shard_log_config *log_cfg, - cache_config *cache_cfg, - data_config *data_cfg) -{ - ZERO_CONTENTS(log_cfg); - log_cfg->cache_cfg = cache_cfg; - log_cfg->data_cfg = data_cfg; - log_cfg->seed = HASH_SEED; - log_cfg->blob_cfg = (blob_build_config){ - .extent_batch = 1, - .page_batch = 2, - .subpage_batch = 3, - .alignment = cache_config_page_size(cache_cfg), - }; -} - -void -shard_log_print(shard_log *log) -{ - cache *cc = log->cc; - uint64 extent_addr = log->addr; - shard_log_config *cfg = log->cfg; - uint64 magic = log->magic; - data_config *dcfg = cfg->data_cfg; - uint64 pages_per_extent = shard_log_pages_per_extent(cfg); - allocator *al = cache_get_allocator(cc); - - while (extent_addr != 0 && allocator_get_refcount(al, extent_addr) > 0) { - cache_prefetch(cc, extent_addr, PAGE_TYPE_LOG); - uint64 next_extent_addr = 0; - for (uint64 i = 0; i < pages_per_extent; i++) { - uint64 page_addr = extent_addr + i * shard_log_page_size(cfg); - page_handle *page = cache_get(cc, page_addr, TRUE, PAGE_TYPE_LOG); - if (shard_log_valid(cfg, page, magic)) { - next_extent_addr = shard_log_next_extent_addr(cfg, page); - for (log_entry *le = first_log_entry(page->data); - !terminal_log_entry(cfg, page->data, le); - le = log_entry_next(le)) - { - platform_default_log( - "%s -- %s%s : %lu\n", - key_string(dcfg, log_entry_key(le)), - log_entry_message_is_blob(le) ? "(blob) " : "", - message_string(dcfg, log_entry_message(cc, le)), - le->generation); - } - } - cache_unget(cc, page); - } - extent_addr = next_extent_addr; + platform_status rc = + shard_log_iterator_init(cc, cfg, hid, head.addr, head.magic, itor); + if (!SUCCESS(rc)) { + platform_error_log("shard_log_iterator_create: shard_log_iterator_init " + "failed: %s\n", + platform_status_to_string(rc)); + platform_free(hid, itor); + return NULL; } + return &itor->super; } diff --git a/src/shard_log.h b/src/shard_log.h index 6459d78d..efc74f8e 100644 --- a/src/shard_log.h +++ b/src/shard_log.h @@ -41,17 +41,26 @@ typedef struct shard_log { log_handle super; // handle to log I/O ops abstraction. cache *cc; shard_log_config *cfg; + platform_heap_id heap_id; shard_log_thread_data thread_data[MAX_THREADS]; mini_allocator mini; uint64 addr; uint64 meta_head; uint64 magic; + /* + * Extents the mini-allocator held once the stream was initialized -- its + * fixed per-stream overhead (a metadata extent plus one per batch). + * shard_log_get_size() subtracts it so a fresh stream reports zero bytes + * appended. + */ + uint64 initial_extents; } shard_log; typedef struct log_entry log_entry; typedef struct shard_log_iterator { - iterator super; + log_iterator super; // IS-A log_iterator IS-A generic iterator + platform_heap_id heap_id; cache *cc; shard_log_config *cfg; char *contents; @@ -73,22 +82,25 @@ typedef struct ONDISK shard_log_hdr { uint16 num_entries; } shard_log_hdr; -platform_status -shard_log_init(shard_log *log, cache *cc, shard_log_config *cfg); - -void -shard_log_zap(shard_log *log); - -platform_status -shard_log_iterator_init(cache *cc, - shard_log_config *cfg, - platform_heap_id hid, - uint64 addr, - uint64 magic, - shard_log_iterator *itor); +/* + * Create a fresh sharded write-ahead log stream. Returns an abstract + * log_handle (or NULL on failure) to be driven through the log.h interface and + * retired with log_seal(). + */ +log_handle * +shard_log_create(cache *cc, shard_log_config *cfg, platform_heap_id hid); -void -shard_log_iterator_deinit(platform_heap_id hid, shard_log_iterator *itor); +/* + * Create an iterator over the sharded log identified by `head`, reading its + * records in generation order. Returns an abstract log_iterator (or NULL on + * failure) to be driven through the log.h interface and freed with + * log_iterator_deinit(). + */ +log_iterator * +shard_log_iterator_create(cache *cc, + shard_log_config *cfg, + platform_heap_id hid, + log_head head); void shard_log_config_init(shard_log_config *log_cfg, diff --git a/src/splinterdb.c b/src/splinterdb.c index b8614668..653f4bd4 100644 --- a/src/splinterdb.c +++ b/src/splinterdb.c @@ -17,6 +17,7 @@ #include "clockcache.h" #include "data_internal.h" #include "rc_allocator.h" +#include "superblock.h" #include "core.h" #include "lookup_result.h" #include "notification.h" @@ -163,6 +164,14 @@ splinterdb_config_set_defaults(splinterdb_config *cfg) if (!cfg->prefetch_budget) { cfg->prefetch_budget = CORE_DEFAULT_PREFETCH_BUDGET; } + if (!cfg->checkpoint_log_size_bytes) { + /* + * Checkpoint once the log has grown by a cache's worth: replaying much + * more log than the cache can hold gains little, since those pages cannot + * stay resident anyway. + */ + cfg->checkpoint_log_size_bytes = cfg->cache_size; + } } static platform_status @@ -243,10 +252,24 @@ splinterdb_init_config(const splinterdb_config *kvs_cfg, // IN allocator_config_init(&kvs->allocator_cfg, &kvs->io_cfg, cfg.disk_size); if (geometry != NULL) { - rc = rc_allocator_disk_geometry_matches_config(geometry, - &kvs->allocator_cfg); - if (!SUCCESS(rc)) { - return rc; + // The auto-read on-disk geometry must agree with the resolved config; + // core_mount re-validates it against the superblock, but check here for + // an early, clear error when the caller overrides a device field. + if (geometry->disk_size != kvs->allocator_cfg.capacity + || geometry->page_size != kvs->io_cfg.page_size + || geometry->extent_size != kvs->io_cfg.extent_size) + { + platform_error_log( + "On-disk geometry (disk_size=%lu page_size=%lu extent_size=%lu) " + "does not match the configured geometry (disk_size=%lu " + "page_size=%lu extent_size=%lu).\n", + geometry->disk_size, + geometry->page_size, + geometry->extent_size, + kvs->allocator_cfg.capacity, + kvs->io_cfg.page_size, + kvs->io_cfg.extent_size); + return STATUS_BAD_PARAM; } } @@ -296,6 +319,7 @@ splinterdb_init_config(const splinterdb_config *kvs_cfg, // IN cfg.queue_scale_percent, cfg.prefetch_budget, cfg.use_log, + cfg.checkpoint_log_size_bytes, cfg.use_stats, FALSE, Platform_default_log_handle); @@ -315,7 +339,7 @@ splinterdb_config_read_disk_geometry(const char *filename, return STATUS_BAD_PARAM; } - platform_status status = rc_allocator_read_disk_geometry(filename, geometry); + platform_status status = superblock_read_geometry(filename, geometry); if (!SUCCESS(status)) { return status; } @@ -473,6 +497,7 @@ splinterdb_create_or_open(const splinterdb_config *kvs_cfg, // IN &kvs->trunk_cfg, (allocator *)&kvs->allocator_handle, (cache *)&kvs->cache_handle, + kvs->io_handle, &kvs->task_sys, kvs->trunk_id, kvs->heap_id); @@ -481,6 +506,7 @@ splinterdb_create_or_open(const splinterdb_config *kvs_cfg, // IN &kvs->trunk_cfg, (allocator *)&kvs->allocator_handle, (cache *)&kvs->cache_handle, + kvs->io_handle, &kvs->task_sys, kvs->trunk_id, kvs->heap_id); @@ -498,7 +524,8 @@ splinterdb_create_or_open(const splinterdb_config *kvs_cfg, // IN deinit_cache: clockcache_deinit(&kvs->cache_handle); deinit_allocator: - rc_allocator_unmount(&kvs->allocator_handle); + /* Initialization/open failed before a successful clean core shutdown. */ + rc_allocator_deinit(&kvs->allocator_handle); deinit_system: task_system_deinit(&kvs->task_sys); deinit_iohandle: @@ -578,7 +605,13 @@ splinterdb_close(splinterdb **kvs_in) // IN } io_wait_all(kvs->io_handle); clockcache_deinit(&kvs->cache_handle); - rc_allocator_unmount(&kvs->allocator_handle); + /* + * core_unmount() already persisted the refcount map and published the + * superblock's allocation state (or, on failure, deliberately left it + * invalid so the next open rebuilds); the allocator now only tears down its + * in-memory structures. + */ + rc_allocator_deinit(&kvs->allocator_handle); task_system_deinit(&kvs->task_sys); io_handle_destroy(kvs->io_handle); diff --git a/src/superblock.c b/src/superblock.c new file mode 100644 index 00000000..09f88ed3 --- /dev/null +++ b/src/superblock.c @@ -0,0 +1,274 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +/* + * superblock.c -- + * + * Implementation of the SplinterDB superblock (see superblock.h). + */ + +#include "superblock.h" +#include "poison.h" + +#define SUPERBLOCK_CSUM_SEED (0x53555045524231ULL) // "SUPERB1" + +static uint64 +superblock_slot_addr(const superblock_context *ctx, uint64 slot) +{ + platform_assert(slot < SUPERBLOCK_NUM_SLOTS); + return slot * ctx->page_size; +} + +static checksum128 +superblock_checksum(const superblock *sb) +{ + return platform_checksum128( + sb, offsetof(superblock, checksum), SUPERBLOCK_CSUM_SEED); +} + +static bool32 +superblock_geometry_matches(disk_geometry geometry, const allocator_config *cfg) +{ + return geometry.disk_size == cfg->capacity + && geometry.page_size == cfg->io_cfg->page_size + && geometry.extent_size == cfg->io_cfg->extent_size; +} + +static bool32 +superblock_is_valid(const superblock *sb, const allocator_config *cfg) +{ + // Copy the packed geometry member by value (avoids an unaligned address). + disk_geometry geometry = sb->geometry; + return sb->format_magic == SUPERBLOCK_FORMAT_MAGIC + && sb->format_version == SUPERBLOCK_FORMAT_VERSION + && superblock_geometry_matches(geometry, cfg) + && platform_checksum_is_equal(sb->checksum, superblock_checksum(sb)); +} + +platform_status +superblock_read_geometry(const char *filename, disk_geometry *geometry) +{ + /* + * Geometry is the first field of the superblock at page 0, and it is + * write-invariant, so a raw read of it is safe even against a torn write of + * the page's mutable fields. + */ + return io_read_bootstrap(filename, geometry, sizeof(*geometry), 0); +} + +platform_status +superblock_context_init(superblock_context *ctx, + io_handle *io, + const allocator_config *cfg, + platform_heap_id hid) +{ + ZERO_CONTENTS(ctx); + ctx->io = io; + ctx->heap_id = hid; + ctx->page_size = cfg->io_cfg->page_size; + + platform_assert(sizeof(superblock) <= ctx->page_size); + + platform_status rc = + platform_buffer_init(&ctx->image_buffer, ctx->page_size); + if (!SUCCESS(rc)) { + return rc; + } + ctx->image = platform_buffer_getaddr(&ctx->image_buffer); + memset(ctx->image, 0, ctx->page_size); + ctx->current_slot = 0; + return STATUS_OK; +} + +void +superblock_context_deinit(superblock_context *ctx) +{ + if (ctx->image != NULL) { + platform_buffer_deinit(&ctx->image_buffer); + ctx->image = NULL; + } +} + +platform_status +superblock_mount(superblock_context *ctx, const allocator_config *cfg) +{ + platform_assert(ctx->image != NULL); + + buffer_handle slot_buffer; + platform_status rc = platform_buffer_init(&slot_buffer, ctx->page_size); + if (!SUCCESS(rc)) { + return rc; + } + superblock *slot = platform_buffer_getaddr(&slot_buffer); + + bool32 have_valid = FALSE; + for (uint64 s = 0; s < SUPERBLOCK_NUM_SLOTS; s++) { + rc = io_read(ctx->io, slot, ctx->page_size, superblock_slot_addr(ctx, s)); + if (!SUCCESS(rc)) { + goto out; + } + if (!superblock_is_valid(slot, cfg)) { + continue; + } + if (!have_valid || slot->generation > ctx->image->generation) { + memcpy(ctx->image, slot, ctx->page_size); + ctx->current_slot = s; + have_valid = TRUE; + } + } + + if (!have_valid) { + platform_error_log("superblock_mount: no valid superblock found\n"); + rc = STATUS_NOT_FOUND; + goto out; + } + rc = STATUS_OK; + +out: +{ + platform_status deinit_rc = platform_buffer_deinit(&slot_buffer); + if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { + rc = deinit_rc; + } +} + return rc; +} + +/* + * Checksum the in-memory image and write it to a physical slot. Does not make + * it durable; superblock_make_durable() issues the barrier. + */ +static platform_status +superblock_write_slot(superblock_context *ctx, uint64 slot) +{ + ctx->image->checksum = superblock_checksum(ctx->image); + return io_write( + ctx->io, ctx->image, ctx->page_size, superblock_slot_addr(ctx, slot)); +} + +platform_status +superblock_make_durable(superblock_context *ctx) +{ + platform_assert(ctx->image != NULL); + + uint64 target = ctx->current_slot ^ 1; + ctx->image->generation += 1; + platform_assert(ctx->image->generation != 0); // generation wraparound + + platform_status rc = superblock_write_slot(ctx, target); + if (!SUCCESS(rc)) { + return rc; + } + rc = io_durable_barrier(ctx->io); + if (!SUCCESS(rc)) { + return rc; + } + ctx->current_slot = target; + return STATUS_OK; +} + +platform_status +superblock_format(superblock_context *ctx, const allocator_config *cfg) +{ + platform_assert(ctx->image != NULL); + + memset(ctx->image, 0, ctx->page_size); + ctx->image->geometry.disk_size = cfg->capacity; + ctx->image->geometry.page_size = cfg->io_cfg->page_size; + ctx->image->geometry.extent_size = cfg->io_cfg->extent_size; + ctx->image->format_magic = SUPERBLOCK_FORMAT_MAGIC; + ctx->image->format_version = SUPERBLOCK_FORMAT_VERSION; + ctx->image->generation = 0; + ctx->image->allocation_state_addr = 0; // fresh DB: rebuild on crash + // A fresh, empty tree: no root, and nothing incorporated yet -- replay from + // generation 0. + ctx->image->tree.first_unincorporated_generation = 0; + + /* + * Write both physical copies so torn-write protection is in force from the + * outset. The first publish targets slot 0 (generation 1) because the + * initial current_slot is 1; the second targets slot 1 (generation 2), + * leaving slot 1 newest. + */ + ctx->current_slot = 1; + platform_status rc = superblock_make_durable(ctx); + if (!SUCCESS(rc)) { + return rc; + } + return superblock_make_durable(ctx); +} + +bool32 +superblock_allocation_state_valid(const superblock_context *ctx) +{ + return ctx->image->allocation_state_addr != 0; +} + +uint64 +superblock_allocation_state_addr(const superblock_context *ctx) +{ + return ctx->image->allocation_state_addr; +} + +void +superblock_get_tree_record(const superblock_context *ctx, + superblock_tree_record *out) +{ + *out = ctx->image->tree; +} + +void +superblock_log_cut(superblock_context *ctx, superblock_log_head new_live) +{ + // The retiring live log becomes the sealed log: its entries are still being + // folded into the next root. new_live receives subsequent inserts. The + // persisted allocation map no longer matches the (log) state, so invalidate + // it. + ctx->image->tree.sealed_log = ctx->image->tree.live_log; + ctx->image->tree.live_log = new_live; + ctx->image->allocation_state_addr = 0; +} + +void +superblock_snapshot_tree(superblock_context *ctx, + uint64 root_addr, + uint64 first_unincorporated_generation) +{ + // Advancing the root diverges the persisted allocation map, so invalidate + // it. The live log is left alone; see the header. + ctx->image->tree.root_addr = root_addr; + ctx->image->tree.first_unincorporated_generation = + first_unincorporated_generation; + ctx->image->allocation_state_addr = 0; + + /* + * Drop the sealed log only once the new root covers everything it holds. It + * covers generations up to live_log.start_generation - 1, so it is fully + * incorporated exactly when first_unincorporated_generation reaches + * live_log.start_generation. Otherwise keep it: recovery still needs those + * entries. + */ + if (first_unincorporated_generation + >= ctx->image->tree.live_log.start_generation) + { + ctx->image->tree.sealed_log = (superblock_log_head){0}; + } +} + +void +superblock_discard_logs(superblock_context *ctx) +{ + ctx->image->tree.sealed_log = (superblock_log_head){0}; + ctx->image->tree.live_log = (superblock_log_head){0}; + ctx->image->allocation_state_addr = 0; +} + +void +superblock_snapshot_allocator(superblock_context *ctx, uint64 map_addr) +{ + // The only operation that validates the allocation state; every tree/log + // transition invalidates it. The caller must have made the map itself + // durable first, then make the superblock durable after. + platform_assert(map_addr != 0); + ctx->image->allocation_state_addr = map_addr; +} diff --git a/src/superblock.h b/src/superblock.h new file mode 100644 index 00000000..16922a32 --- /dev/null +++ b/src/superblock.h @@ -0,0 +1,264 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +/* + * superblock.h -- + * + * The SplinterDB superblock: the single, atomically-updated root of all + * durable instance metadata. It unifies what used to be three unrelated + * mechanisms (the allocator meta page, the allocator clean-state A/B + * records, and core's per-table checkpoint directory + record A/B pairs) + * into one structure, so that a checkpoint advances a tree's root and + * invalidates the persisted allocation state in a single atomic write. + * + * Two physical copies live at pages 0 and 1 of the device. Writes + * alternate between them and carry a monotonic generation number, so a + * torn write always leaves the previous generation intact in the other + * page. Mount picks the newest copy that passes magic/version/checksum/ + * geometry validation. + * + * The disk geometry is the first field (offset 0) so it can be read via a + * raw bootstrap read before any page size is known. Geometry is + * write-invariant -- every generation writes identical geometry bytes -- + * so a torn write to page 0 can only damage the checksummed mutable fields + * (recovered from the other page), never the geometry the bootstrap read + * depends on. + * + * The instance holds a single tree today: its per-tree record is embedded + * directly rather than as an array. Supporting multiple trees would be an + * on-disk format change (version bump + reformat), not just a machinery + * change. + */ + +#pragma once + +#include "platform_hash.h" +#include "platform_buffer.h" +#include "allocator.h" +#include "platform_io.h" +#include "util.h" + +#define SUPERBLOCK_FORMAT_MAGIC (0x5344425355504552ULL) // SDBSUPER +/* v2 added superblock_log_head.start_generation. */ +#define SUPERBLOCK_FORMAT_VERSION (2) + +/* The two physical superblock copies live at pages 0 and 1. */ +#define SUPERBLOCK_NUM_SLOTS (2) + +/* + * A log's on-disk head, plus the range of memtable generations it covers. The + * addr/meta_addr/magic triple mirrors log_head's layout; the superblock stores + * it opaquely and does not depend on the log module. meta_addr == 0 means "no + * log present". + */ +typedef struct ONDISK superblock_log_head { + uint64 addr; + uint64 meta_addr; + uint64 magic; + /* + * First memtable generation whose entries this log received. A log's + * coverage ends where the next log's begins, so the sealed log covers + * [sealed_log.start_generation, live_log.start_generation - 1]. That lets + * the superblock decide on its own when a sealed log has been fully + * incorporated and can be dropped (see superblock_snapshot_tree()), and it + * tells recovery which log to begin replaying from. + */ + uint64 start_generation; +} superblock_log_head; + +/* An empty (absent) log slot: meta_addr == 0. */ +#define SUPERBLOCK_NO_LOG(info) ((info).meta_addr == 0) + +/* + * The durable per-tree record. The instance always has exactly one tree (from + * mkfs onward), so the record carries no id or occupancy marker. + * + * The two log pointers implement the two-log checkpoint protocol: live_log is + * the stream currently receiving inserts; sealed_log is set only while a + * checkpoint is in progress -- the just-sealed stream whose entries are being + * folded into the new root. At rest (between checkpoints, and after a clean + * unmount) sealed_log is empty. On crash recovery the sealed log (if present) + * then the live log are replayed onto root_addr, replaying entries at or above + * first_unincorporated_generation. + * + * There is intentionally no clean/dirty ("unmounted") flag: replaying a clean + * root is a no-op because every entry is below first_unincorporated_generation, + * so the superblock's allocation_state_addr validity is the single at-rest + * signal. + */ +typedef struct ONDISK superblock_tree_record { + uint64 root_addr; + /* + * First memtable generation NOT folded into root_addr -- the exclusive + * replay bound and the generation to resume at on mount. 0 means nothing + * has been incorporated yet (replay from the start); no sentinel is needed + * because generation 0 being unincorporated is the natural fresh state. + */ + uint64 first_unincorporated_generation; + superblock_log_head live_log; + superblock_log_head sealed_log; +} superblock_tree_record; + +typedef struct ONDISK superblock { + disk_geometry geometry; // MUST be first; see the bootstrap-read note above. + uint64 format_magic; + uint64 format_version; + uint64 generation; // monotonic; newest valid copy wins at mount + /* + * Base address of the persisted allocator refcount map when it is + * trustworthy, or 0 to force a rebuild-by-walking on the next mount. A + * checkpoint writes this as 0 in the same atomic update that advances a + * root; a clean unmount writes it nonzero after the map is durable. + */ + uint64 allocation_state_addr; + superblock_tree_record tree; + checksum128 checksum; +} superblock; + +_Static_assert(offsetof(superblock, geometry) == 0, + "disk geometry must be the first superblock field"); +_Static_assert(sizeof(superblock) <= IO_DEFAULT_PAGE_SIZE, + "superblock must fit in the default page size"); + +/* + * In-memory handle over the on-disk superblock. Holds a page-aligned image of + * the current superblock and tracks which physical slot (0 or 1) it came from, + * so a publish can target the other slot. + */ +typedef struct superblock_context { + io_handle *io; + platform_heap_id heap_id; + uint64 page_size; + buffer_handle image_buffer; + superblock *image; // page-aligned, page-sized + uint64 current_slot; // slot the in-memory image was last read/written +} superblock_context; + +/* + * Read the raw disk geometry via a bootstrap read of page 0, before any + * subsystem is configured. Safe against torn writes because geometry is + * write-invariant. Does not validate the checksum (the full mount does). + */ +platform_status +superblock_read_geometry(const char *filename, disk_geometry *geometry); + +/* Allocate the in-memory image. Does no I/O. */ +platform_status +superblock_context_init(superblock_context *ctx, + io_handle *io, + const allocator_config *cfg, + platform_heap_id hid); + +/* Release in-memory resources. Does no I/O. Specifically, does _not_ make the + * current in-memory superblock durable. */ +void +superblock_context_deinit(superblock_context *ctx); + +/* + * Read both physical copies, validate each (magic, version, checksum, and + * geometry against cfg), and load the newest valid one into the in-memory + * image. Returns STATUS_NOT_FOUND if neither copy is a valid superblock. + * + * Must call superblock_init() first. + */ +platform_status +superblock_mount(superblock_context *ctx, const allocator_config *cfg); + +/* + * Initialize a fresh superblock in the in-memory image (empty tree table, + * allocation state invalid) and write both physical copies durably. Used by + * mkfs. + * + * Must call superblock_init() first. + */ +platform_status +superblock_format(superblock_context *ctx, const allocator_config *cfg); + +/* + * ---- Durable-state transitions ---- + * + * Each transition mutates the in-memory image only; nothing reaches disk until + * superblock_make_durable(). A checkpoint or clean unmount is a sequence of + * these transitions followed by one superblock_make_durable(), so the + * durability boundary is always explicit at the call site. + * + * The tree/log transitions encode the crash-safety invariant that the persisted + * allocation map is trustworthy only while it matches the durable tree: + * superblock_log_cut() and superblock_snapshot_tree() both invalidate it, and + * superblock_snapshot_allocator() is the only operation that re-validates it. + */ + +/* + * Cut the log: the current live log becomes the sealed log -- its entries are + * still being folded into the next root -- and new_live receives subsequent + * inserts. Invalidates the allocation state. Used at a checkpoint's begin, + * and to install this session's log at mkfs/mount (where the current live log + * is empty, so the sealed slot stays empty). + * + * This is the only operation that installs a log, which is what lets it derive + * the sealed log from the image rather than taking it on trust. + */ +void +superblock_log_cut(superblock_context *ctx, superblock_log_head new_live); + +/* + * Advance the durable tree to root_addr, where first_unincorporated_generation + * is the first generation not folded into it. Invalidates the allocation + * state. Used at a checkpoint's completion, at a durability checkpoint, and at + * a clean unmount. + * + * Deliberately does not touch the live log: which log is live is the cut + * protocol's business (superblock_log_cut()), and a root advance that rewrote + * it could orphan a log still holding unincorporated entries. + * + * The sealed log is dropped only once it is fully incorporated, which this + * decides on its own: the sealed log covers generations up to + * live_log.start_generation - 1, so it is droppable exactly when + * first_unincorporated_generation >= live_log.start_generation. Otherwise it + * is preserved, because recovery would still need it. Callers therefore cannot + * drop a sealed log prematurely. + */ +void +superblock_snapshot_tree(superblock_context *ctx, + uint64 root_addr, + uint64 first_unincorporated_generation); + +/* + * Drop both log slots. Used at a clean shutdown, which has folded everything + * into the durable root and frees the log's extents, so no log may remain + * referenced. Invalidates the allocation state. + */ +void +superblock_discard_logs(superblock_context *ctx); + +/* + * Record the persisted allocator refcount map at map_addr as trustworthy. This + * is the only operation that validates the allocation state; the caller must + * have made the map itself durable first. Used as the final step of a clean + * unmount. + */ +void +superblock_snapshot_allocator(superblock_context *ctx, uint64 map_addr); + +/* + * Make the current in-memory image durable: bump the generation, write it to + * the physical slot not currently newest, and issue a durable barrier. On + * success that slot becomes newest; a torn write leaves the previous generation + * intact in the other slot. This is the single durability boundary for the + * transitions above. + */ +platform_status +superblock_make_durable(superblock_context *ctx); + +/* ---- Read-only accessors on the in-memory image ---- */ + +bool32 +superblock_allocation_state_valid(const superblock_context *ctx); + +uint64 +superblock_allocation_state_addr(const superblock_context *ctx); + +/* Copy the (always-present) tree record into *out. */ +void +superblock_get_tree_record(const superblock_context *ctx, + superblock_tree_record *out); diff --git a/src/trunk.c b/src/trunk.c index 133139a5..aa1eb730 100644 --- a/src/trunk.c +++ b/src/trunk.c @@ -2008,7 +2008,6 @@ trunk_node_serialize_maybe_setup_next_page(cache *cc, "%s():%d: cache_alloc() failed", __func__, __LINE__); return STATUS_NO_MEMORY; } - cache_mark_dirty(cc, *current_page); *page_offset = 0; } @@ -2131,7 +2130,6 @@ trunk_node_serialize(trunk_context *context, trunk_node *node) rc = STATUS_NO_MEMORY; goto cleanup; } - cache_mark_dirty(context->cc, header_page); int64 min_inflight_bundle_start = trunk_node_first_live_inflight_bundle(node); @@ -2541,6 +2539,59 @@ trunk_read_end(trunk_context *context) batch_rwlock_unget(&context->root_lock, 0); } +/* + * Capture the root address and acquire its allocator reference while the root + * lock prevents COW publication from dropping the live reference. This is + * deliberately metadata-only: checkpointing must not fault the root page just + * to take a snapshot. + */ +platform_status +trunk_snapshot_create(trunk_context *context, trunk_snapshot *snapshot) +{ + snapshot->root_addr = 0; + + trunk_read_begin(context); + if (context->root != NULL) { + snapshot->root_addr = context->root->ref.addr; + trunk_addr_inc_ref(context->al, snapshot->root_addr); + } + trunk_read_end(context); + + return STATUS_OK; +} + +platform_status +trunk_snapshot_create_from_addr(allocator *al, + uint64 root_addr, + trunk_snapshot *snapshot) +{ + snapshot->root_addr = root_addr; + if (root_addr != 0) { + trunk_addr_inc_ref(al, root_addr); + } + return STATUS_OK; +} + +platform_status +trunk_snapshot_release(trunk_context *context, trunk_snapshot *snapshot) +{ + if (snapshot->root_addr == 0) { + return STATUS_OK; + } + + /* + * Perform the single decrement the snapshot's reference is owed against + * `context` itself. It must not be routed through a temporary context: if + * the root is still cache_in_use (a concurrent reader holds it), the + * decrement defers the node's destruction onto context->pending_gcs, and + * only a context that outlives this call will ever drain that list. + */ + uint64 root_addr = snapshot->root_addr; + snapshot->root_addr = 0; + trunk_ondisk_node_dec_ref(context, root_addr); + return STATUS_OK; +} + platform_status trunk_init_root_handle(trunk_context *context, trunk_ondisk_node_handle *handle) { @@ -6705,7 +6756,7 @@ trunk_context_init(trunk_context *context, cache *cc, allocator *al, task_system *ts, - uint64 root_addr) + trunk_snapshot snapshot) { memset(context, 0, sizeof(trunk_context)); @@ -6715,12 +6766,20 @@ trunk_context_init(trunk_context *context, context->al = al; context->ts = ts; - if (root_addr != 0) { + if (snapshot.root_addr != 0) { + // Adopt: snapshot already carries an owned reference (from + // trunk_snapshot_create or trunk_snapshot_create_from_addr), so this + // does not take a new one; it just gives that reference a home. context->root = trunk_ondisk_node_ref_create( - context, NEGATIVE_INFINITY_KEY, root_addr, FALSE); + context, NEGATIVE_INFINITY_KEY, snapshot.root_addr, TRUE /* adopt */); if (context->root == NULL) { platform_error_log("trunk_node_context_init: " "ondisk_node_ref_create failed\n"); + // The reference was never actually adopted (creation failed before + // that could happen), but the caller has already relinquished it. + // Discharge it here so this function always consumes its snapshot, + // regardless of outcome. + trunk_ondisk_node_dec_ref(context, snapshot.root_addr); return STATUS_NO_MEMORY; } } @@ -6746,34 +6805,6 @@ trunk_context_init(trunk_context *context, return STATUS_OK; } -void -trunk_inc_ref(allocator *al, uint64 root_addr) -{ - trunk_addr_inc_ref(al, root_addr); -} - -platform_status -trunk_dec_ref(const trunk_config *cfg, - platform_heap_id hid, - cache *cc, - allocator *al, - task_system *ts, - uint64 root_addr) -{ - trunk_context context; - platform_status rc = - trunk_context_init(&context, cfg, hid, cc, al, ts, root_addr); - if (!SUCCESS(rc)) { - platform_error_log("trunk_node_dec_ref: trunk_node_context_init failed: " - "%d\n", - rc.r); - return rc; - } - trunk_ondisk_node_dec_ref(&context, root_addr); - trunk_context_deinit(&context); - return STATUS_OK; -} - void trunk_context_deinit(trunk_context *context) { @@ -6790,34 +6821,6 @@ trunk_context_deinit(trunk_context *context) } } - -platform_status -trunk_context_clone(trunk_context *dst, trunk_context *src) -{ - platform_status rc; - trunk_ondisk_node_handle handle; - rc = trunk_init_root_handle(src, &handle); - if (!SUCCESS(rc)) { - platform_error_log("trunk_node_context_clone: trunk_init_root_handle " - "failed: %d\n", - rc.r); - return rc; - } - uint64 root_addr = handle.header_page->disk_addr; - - rc = trunk_context_init( - dst, src->cfg, src->hid, src->cc, src->al, src->ts, root_addr); - trunk_ondisk_node_handle_deinit(&handle); - return rc; -} - -platform_status -trunk_make_durable(trunk_context *context) -{ - cache_flush(context->cc); - return STATUS_OK; -} - /************************************ * Stats ************************************/ diff --git a/src/trunk.h b/src/trunk.h index 7df20d09..70786bec 100644 --- a/src/trunk.h +++ b/src/trunk.h @@ -174,6 +174,15 @@ typedef struct trunk_context { incorporation_tasks tasks; } trunk_context; +/* + * An owned, point-in-time reference to a COW trunk root. The reference keeps + * the root (and therefore its reachable subtree) alive until it is either + * released or transferred to a durable checkpoint record. + */ +typedef struct trunk_snapshot { + uint64 root_addr; +} trunk_snapshot; + typedef struct trunk_ondisk_node_handle { cache *cc; page_handle *header_page; @@ -196,6 +205,14 @@ trunk_config_init(trunk_config *config, uint64 prefetch_budget, bool32 use_stats); +/* + * Initializes context with a root of snapshot. Consumes snapshot's owned + * reference regardless of outcome: on success the reference now lives in + * context (released by a later trunk_context_deinit); on failure it has + * already been discharged. Either way, the caller must not use or release + * snapshot again. A null snapshot (root_addr == 0) starts an empty trunk with + * no root, as for a freshly formatted table. + */ platform_status trunk_context_init(trunk_context *context, const trunk_config *cfg, @@ -203,37 +220,35 @@ trunk_context_init(trunk_context *context, cache *cc, allocator *al, task_system *ts, - uint64 root_addr); + trunk_snapshot snapshot); void -trunk_inc_ref(allocator *al, uint64 root_addr); +trunk_context_deinit(trunk_context *context); +/* Capture an owned reference to the current COW root without reading it. */ platform_status -trunk_dec_ref(const trunk_config *cfg, - platform_heap_id hid, - cache *cc, - allocator *al, - task_system *ts, - uint64 root_addr); +trunk_snapshot_create(trunk_context *context, trunk_snapshot *snapshot); -void -trunk_context_deinit(trunk_context *context); - -/* Create a writable snapshot of a trunk */ +/* + * Capture an owned reference to a root_addr that is not (or not yet) any live + * context's root -- e.g. one just read out of a durable checkpoint record. + * root_addr must already be referenced by the caller (its persisted refcount + * accounts for it); this takes out an additional, independent reference. A + * null root_addr (0) produces a null snapshot without touching the allocator. + */ platform_status -trunk_context_clone(trunk_context *dst, trunk_context *src); +trunk_snapshot_create_from_addr(allocator *al, + uint64 root_addr, + trunk_snapshot *snapshot); -/* Make a trunk durable */ +/* Drop an owned snapshot reference that was not published. */ platform_status -trunk_make_durable(trunk_context *context); +trunk_snapshot_release(trunk_context *context, trunk_snapshot *snapshot); /******************************** * Mutations ********************************/ -void -trunk_modification_begin(trunk_context *context); - // Build a new trunk with the branch incorporated. The new trunk is not yet // visible to queriers. platform_status @@ -259,8 +274,6 @@ trunk_optimize(trunk_context *context, bool32 full_leaf_compactions, struct splinterdb_notification *notification); -void -trunk_modification_end(trunk_context *context); /******************************** * Queries @@ -270,9 +283,6 @@ platform_status trunk_init_root_handle(trunk_context *context, trunk_ondisk_node_handle *handle); -uint64 -trunk_ondisk_node_handle_addr(const trunk_ondisk_node_handle *handle); - void trunk_ondisk_node_handle_deinit(trunk_ondisk_node_handle *handle); diff --git a/test.sh b/test.sh index a8c213ba..fa9fe962 100755 --- a/test.sh +++ b/test.sh @@ -12,6 +12,52 @@ function run() rm -f db } +# Run a workload that is expected to complete at least one checkpoint, and fail +# if it completes none. +# +# A plain run() would pass whether or not checkpointing ever fired, so a +# regression that quietly stopped taking checkpoints would look green. The +# caller must pass --log (to enable the log, and hence auto-checkpointing) and +# --stats (so core_print_insertion_stats() emits the "| checkpoints:" line this +# parses). Several drivers dump stats more than once and the counter is +# cumulative, so take the largest value printed. +function run_checkpointed() +{ + local logfile + logfile=$(mktemp) + + set +e + run_with_timing "$*" ${BINDIR}/$@ $DB_CAPACITY > "$logfile" 2>&1 + local rc=$? + set -e + + cat "$logfile" + rm -f db + + if [ "$rc" -ne 0 ]; then + rm -f "$logfile" + echo "FAILED: $* exited with status $rc" + exit 1 + fi + + local checkpoints + checkpoints=$(grep -oE '\| checkpoints: +[0-9]+' "$logfile" \ + | grep -oE '[0-9]+' | sort -rn | head -1) + rm -f "$logfile" + + if [ -z "$checkpoints" ]; then + echo "FAILED: $* printed no checkpoint statistics" \ + "(both --log and --stats are required)" + exit 1 + fi + if [ "$checkpoints" -eq 0 ]; then + echo "FAILED: $* completed 0 checkpoints;" \ + "lower --checkpoint-log-size-mib or grow the workload" + exit 1 + fi + echo "PASSED: $* completed ${checkpoints} checkpoints" +} + # 14 minutes function cache_tests_1() { # 25 sec each @@ -39,19 +85,51 @@ function cache_tests_3() { # 12 minutes function functionality_tests() { # 50 sec each - run driver_test splinter_test --functionality 1000000 100 --seed 135 - run driver_test splinter_test --functionality 1000000 100 --num-normal-bg-threads 4 --num-memtable-bg-threads 2 --seed 135 - run driver_test splinter_test --functionality 1000000 100 --key-size 102 --seed 135 - run driver_test splinter_test --functionality 1000000 100 --key-size 8 --seed 135 - run driver_test splinter_test --functionality 1000000 100 --use-shmem --seed 135 - run driver_test splinter_test --functionality 1000000 100 --use-shmem --num-normal-bg-threads 4 --num-memtable-bg-threads 2 --seed 135 - run driver_test splinter_test --functionality 1000000 100 --use-shmem --key-size 102 --seed 135 - run driver_test splinter_test --functionality 1000000 100 --use-shmem --key-size 8 --seed 135 - run driver_test splinter_test --functionality 1000000 1000 --num-tables 2 --cache-capacity-mib 1024 - run driver_test splinter_test --functionality 1000000 1000 --num-tables 4 --cache-capacity-mib 1024 - run driver_test splinter_test --functionality 1000000 1000 --num-tables 4 --cache-capacity-mib 512 - run driver_test splinter_test --functionality 10000000 1000 --num-tables 1 --cache-capacity-mib 4096 - run driver_test splinter_test --functionality 10000000 1000 --num-tables 2 --cache-capacity-mib 4096 + run driver_test splinter_test --functionality 1000000 100 --seed 135 + run driver_test splinter_test --functionality 1000000 100 --num-normal-bg-threads 4 --num-memtable-bg-threads 2 --seed 135 + run driver_test splinter_test --functionality 1000000 100 --key-size 102 --seed 135 + run driver_test splinter_test --functionality 1000000 100 --key-size 8 --seed 135 + run driver_test splinter_test --functionality 1000000 100 --use-shmem --seed 135 + run driver_test splinter_test --functionality 1000000 100 --use-shmem --num-normal-bg-threads 4 --num-memtable-bg-threads 2 --seed 135 + run driver_test splinter_test --functionality 1000000 100 --use-shmem --key-size 102 --seed 135 + run driver_test splinter_test --functionality 1000000 100 --use-shmem --key-size 8 --seed 135 + run driver_test splinter_test --functionality 1000000 1000 --cache-capacity-mib 1024 + run driver_test splinter_test --functionality 1000000 1000 --cache-capacity-mib 512 + run driver_test splinter_test --functionality 10000000 1000 --cache-capacity-mib 4096 +} + +# 8 minutes +# +# Logging (and hence auto-checkpointing) is off by default in every other group +# here, so without these the checkpoint machinery goes essentially unexercised +# outside the unit tests. --checkpoint-log-size-mib is deliberately far below +# the production default (one cache's worth) so that checkpoints fire many times +# during a short run; run_checkpointed() asserts they actually did. +function checkpoint_tests() { + # 45 sec each. Correctness under checkpointing: the shadow-verified + # workload, which will catch a checkpoint that corrupts or loses data. + run_checkpointed driver_test splinter_test --functionality 500000 100 --seed 135 --log --checkpoint-log-size-mib 2 --stats + run_checkpointed driver_test splinter_test --functionality 500000 100 --use-shmem --seed 135 --log --checkpoint-log-size-mib 2 --stats + + # 60 sec. Concurrent inserts/lookups/range-lookups across a log cut. + run_checkpointed driver_test splinter_test --perf --max-async-inflight 0 --num-insert-threads 4 --num-lookup-threads 4 --num-range-lookup-threads 4 --num-inserts 300000 --cache-capacity-mib 512 --log --checkpoint-log-size-mib 2 --stats --num-normal-bg-threads 2 --num-memtable-bg-threads 2 + + # 90 sec each. Regression coverage for the pending_gcs crash: a checkpoint + # retiring the old root while an async lookup still pins it in the cache + # deferred the node's destruction onto a scratch context that was freed + # immediately after. Needs async lookups AND frequent checkpoints together. + run_checkpointed driver_test splinter_test --parallel-perf --max-async-inflight 10 --num-pthreads 8 --tree-size-mib 512 --num-normal-bg-threads 4 --num-memtable-bg-threads 2 --log --checkpoint-log-size-mib 2 --stats + run_checkpointed driver_test splinter_test --parallel-perf --max-async-inflight 0 --num-pthreads 8 --lookup-positive-percent 10 --tree-size-mib 512 --log --checkpoint-log-size-mib 2 --stats + + # 45 sec. Deletes and repeated overwrite rounds; the overwrite workload is + # the one that never advances the memtable generation, so it exercises the + # log-size trigger rather than the generation-count one. + run_checkpointed driver_test splinter_test --delete --tree-size-mib 512 --log --checkpoint-log-size-mib 2 --stats + run_checkpointed driver_test splinter_test --periodic --tree-size-mib 256 --log --checkpoint-log-size-mib 2 --stats + + # 60 sec. A realistic cadence rather than a pathological one, so the + # coverage does not depend solely on a tiny threshold. + run_checkpointed driver_test splinter_test --parallel-perf --max-async-inflight 10 --num-pthreads 8 --tree-size-mib 512 --num-normal-bg-threads 4 --num-memtable-bg-threads 2 --log --checkpoint-log-size-mib 64 --stats } function parallel_perf_test_1() { @@ -176,6 +254,7 @@ function all_tests() { cache_tests_2 cache_tests_3 functionality_tests + checkpoint_tests parallel_perf_test_1 parallel_perf_test_2 parallel_perf_test_3 @@ -193,7 +272,6 @@ function all_tests() { } function main() { - echo > db.sizes.log if [ -z "$TESTS_FUNCTION" ]; then TESTS_FUNCTION="all_tests" fi diff --git a/tests/config.c b/tests/config.c index 1f2f873d..6123bb1c 100644 --- a/tests/config.c +++ b/tests/config.c @@ -83,6 +83,8 @@ config_set_defaults(master_config *cfg) .filter_hash_size = TEST_CONFIG_DEFAULT_FILTER_HASH_SIZE, .filter_log_index_size = TEST_CONFIG_DEFAULT_FILTER_LOG_INDEX_SIZE, .use_log = FALSE, + // 0 == follow cache_capacity; see master_config. + .checkpoint_log_size = 0, .num_normal_bg_threads = TEST_CONFIG_DEFAULT_NUM_NORMAL_BG_THREADS, .num_memtable_bg_threads = TEST_CONFIG_DEFAULT_NUM_MEMTABLE_BG_THREADS, .memtable_capacity = MiB_TO_B(TEST_CONFIG_DEFAULT_MEMTABLE_CAPACITY_MB), @@ -153,6 +155,9 @@ config_usage() platform_error_log("\t--no-stats\n"); platform_error_log("\t--log\n"); platform_error_log("\t--no-log\n"); + platform_error_log("\t--checkpoint-log-size-gib\n"); + platform_error_log("\t--checkpoint-log-size-mib\n"); + platform_error_log("\t--checkpoint-log-size-bytes (0 => cache capacity)\n"); platform_error_log("\t--verbose-logging\n"); platform_error_log("\t--no-verbose-logging\n"); platform_error_log("\t--verbose-progress\n"); @@ -384,6 +389,12 @@ config_parse(master_config *cfg, const uint8 num_config, int argc, char *argv[]) config_set_mib("prefetch-budget", cfg, prefetch_budget) {} config_set_gib("prefetch-budget", cfg, prefetch_budget) {} config_set_uint64("prefetch-budget-bytes", cfg, prefetch_budget) {} + config_set_mib("checkpoint-log-size", cfg, checkpoint_log_size) {} + config_set_gib("checkpoint-log-size", cfg, checkpoint_log_size) {} + config_set_uint64( + "checkpoint-log-size-bytes", cfg, checkpoint_log_size) + { + } config_set_mib("memtable-capacity", cfg, memtable_capacity) {} config_set_gib("memtable-capacity", cfg, memtable_capacity) {} config_set_uint64("rough-count-height", cfg, btree_rough_count_height) diff --git a/tests/config.h b/tests/config.h index 089f06a1..8fbdb8af 100644 --- a/tests/config.h +++ b/tests/config.h @@ -75,6 +75,14 @@ typedef struct master_config { // log bool32 use_log; + /* + * Take an automatic checkpoint once the live log has grown by this many + * bytes. Zero means follow cache_capacity, matching the production default + * of one cache's worth of log; it is resolved that way at use, so lowering + * --cache-capacity also lowers this unless it is set explicitly. Set it + * very large (UINT64_MAX) to effectively disable automatic checkpoints. + */ + uint64 checkpoint_log_size; // task system uint64 num_normal_bg_threads; // Both bg_threads fields have to be non-zero diff --git a/tests/functional/btree_test.c b/tests/functional/btree_test.c index 2511e615..32b265e2 100644 --- a/tests/functional/btree_test.c +++ b/tests/functional/btree_test.c @@ -17,6 +17,7 @@ #include "rc_allocator.h" #include "cache.h" #include "clockcache.h" +#include "mini_allocator.h" #include "random.h" #include "task.h" @@ -70,6 +71,95 @@ test_btree_process_noop(void *arg, uint64 generation) // really a no-op } +static platform_status +test_memtable_generation_init(cache *cc, + test_btree_config *cfg, + platform_heap_id hid) +{ + const uint64 first_generation = 17; + const uint64 max_memtables = 4; + memtable_config mt_cfg = *cfg->mt_cfg; + memtable_context mt_ctxt; + + mt_cfg.max_memtables = max_memtables; + + platform_status rc = memtable_context_init( + &mt_ctxt, hid, cc, &mt_cfg, NULL, test_btree_process_noop, NULL); + if (!SUCCESS(rc)) { + return rc; + } + + if (mt_ctxt.generation != 0 || mt_ctxt.generation_to_incorporate != 0 + || mt_ctxt.generation_retired != (uint64)-1) + { + platform_error_log("memtable fresh initialization has unexpected " + "generation state\n"); + rc = STATUS_TEST_FAILED; + goto deinit_fresh; + } + + for (uint64 generation = 0; generation < max_memtables; generation++) { + uint64 mt_no = generation % max_memtables; + if (mt_ctxt.mt[mt_no].generation != generation) { + platform_error_log("memtable fresh initialization put generation " + "%lu in the wrong slot\n", + generation); + rc = STATUS_TEST_FAILED; + goto deinit_fresh; + } + } + +deinit_fresh: + memtable_context_deinit(&mt_ctxt); + if (!SUCCESS(rc)) { + return rc; + } + + rc = memtable_context_init_at_generation(&mt_ctxt, + hid, + cc, + &mt_cfg, + NULL, + test_btree_process_noop, + NULL, + first_generation); + if (!SUCCESS(rc)) { + return rc; + } + + if (mt_ctxt.generation != first_generation + || mt_ctxt.generation_to_incorporate != first_generation + || mt_ctxt.generation_retired != first_generation - 1) + { + platform_error_log("memtable recovery initialization has unexpected " + "generation state\n"); + rc = STATUS_TEST_FAILED; + goto deinit_recovery; + } + + for (uint64 generation = first_generation; + generation < first_generation + max_memtables; + generation++) + { + uint64 mt_no = generation % max_memtables; + if (mt_ctxt.mt[mt_no].generation != generation) { + platform_error_log("memtable recovery initialization put generation " + "%lu in the wrong slot\n", + generation); + rc = STATUS_TEST_FAILED; + goto deinit_recovery; + } + } + +deinit_recovery: + memtable_context_deinit(&mt_ctxt); + if (SUCCESS(rc)) { + platform_default_log( + "btree_test: memtable generation init test passed\n"); + } + return rc; +} + test_memtable_context * test_memtable_context_create(cache *cc, test_btree_config *cfg, @@ -81,8 +171,13 @@ test_memtable_context_create(cache *cc, ctxt->cc = cc; ctxt->cfg = cfg; ctxt->heap_id = hid; - platform_status rc = memtable_context_init( - &ctxt->mt_ctxt, hid, cc, cfg->mt_cfg, test_btree_process_noop, NULL); + platform_status rc = memtable_context_init(&ctxt->mt_ctxt, + hid, + cc, + cfg->mt_cfg, + NULL, + test_btree_process_noop, + NULL); if (!SUCCESS(rc)) { platform_free(hid, ctxt); return NULL; @@ -2258,6 +2353,9 @@ btree_test(int argc, char *argv[]) platform_assert_status_ok(rc); cache *ccp = (cache *)cc; + rc = test_memtable_generation_init(ccp, &test_cfg, hid); + platform_assert_status_ok(rc); + uint64 max_tuples_per_memtable = test_cfg.mt_cfg->max_extents_per_memtable * cache_config_extent_size((cache_config *)&system_cfg.cache_cfg) / 3 diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index ad4b3b27..e774f90a 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -12,6 +12,7 @@ #include "rc_allocator.h" #include "cache.h" #include "clockcache.h" +#include "mini_allocator.h" #include "task.h" #include "random.h" #include "test_common.h" @@ -90,6 +91,947 @@ cache_test_alloc_extents(cache *cc, return rc; } +static platform_status +cache_test_fill_page(cache *cc, uint64 addr, uint64 page_size, uint8 value) +{ + page_handle *page = cache_get(cc, addr, TRUE, PAGE_TYPE_MISC); + if (!cache_try_claim(cc, page)) { + cache_unget(cc, page); + return STATUS_BUSY; + } + + cache_lock(cc, page); + memset(page->data, value, page_size); + cache_unlock(cc, page); + cache_unclaim(cc, page); + cache_unget(cc, page); + return STATUS_OK; +} + +static platform_status +cache_test_verify_disk_page(cache *cc, + uint64 addr, + uint64 page_size, + uint8 expected) +{ + clockcache *clock = (clockcache *)cc; + buffer_handle buffer; + platform_status rc = platform_buffer_init(&buffer, page_size); + if (!SUCCESS(rc)) { + return rc; + } + + rc = io_read(clock->io, platform_buffer_getaddr(&buffer), page_size, addr); + if (SUCCESS(rc)) { + const uint8 *bytes = platform_buffer_getaddr(&buffer); + for (uint64 i = 0; i < page_size; i++) { + if (bytes[i] != expected) { + platform_error_log("cache_test: unexpected byte at addr=%lu " + "offset=%lu: got=%u expected=%u\n", + addr, + i, + bytes[i], + expected); + rc = STATUS_TEST_FAILED; + break; + } + } + } + + platform_status deinit_rc = platform_buffer_deinit(&buffer); + if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { + rc = deinit_rc; + } + return rc; +} + +/* + * The refcount allocator is a pure store: mount() attaches without reading the + * persisted map, and the caller then chooses how to populate it via + * allocator_load_refcounts(). rebuild == FALSE loads the trusted persisted + * map; rebuild == TRUE ignores it and reserves only the allocator's own fixed + * extents, leaving all other ownership for a caller-driven rebuild. + * persist() writes the map durably; if a rebuild fails there is nothing to + * undo -- deinit() alone discards it, since nothing has been persisted. This + * exercises that contract end to end (clean-vs-rebuild is the caller's + * decision -- the superblock's -- not the allocator's). + */ +static platform_status +test_rc_allocator_recovery_bootstrap(allocator_config *cfg, + io_handle *io, + platform_heap_id hid) +{ + uint64 refcount_buffer_size = + ROUNDUP(cfg->extent_capacity * sizeof(refcount), cfg->io_cfg->page_size); + uint64 refcount_extent_count = + (refcount_buffer_size + cfg->io_cfg->extent_size - 1) + / cfg->io_cfg->extent_size; + // Extent 0 (superblock) + the refcount-map extents. + uint64 reserved_extent_count = 1 + refcount_extent_count; + uint64 stale_extent_addr = 0; + platform_status rc = STATUS_OK; + rc_allocator al; + bool32 al_live = FALSE; + + ZERO_CONTENTS(&al); + platform_default_log( + "cache_test: allocator recovery bootstrap test started\n"); + + // 1) Format, allocate a data extent, persist the map, tear down. + rc = rc_allocator_init(&al, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + al_live = TRUE; + rc = allocator_alloc((allocator *)&al, &stale_extent_addr, PAGE_TYPE_MISC); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = allocator_persist((allocator *)&al, NULL); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc_allocator_deinit(&al); + al_live = FALSE; + + // 2) Clean open: attach + load_refcounts(rebuild=FALSE) loads the map. + rc = rc_allocator_mount(&al, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + al_live = TRUE; + rc = allocator_load_refcounts((allocator *)&al); + if (!SUCCESS(rc)) { + goto cleanup; + } + if (allocator_get_refcount((allocator *)&al, stale_extent_addr) + != AL_ONE_REF) + { + platform_error_log( + "cache_test: clean open did not load the persisted refcount map\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc_allocator_deinit(&al); + al_live = FALSE; + + // 3) Rebuild open: attach + load_refcounts(rebuild=TRUE) ignores the + // persisted map, reserving only the fixed extents. + rc = rc_allocator_mount(&al, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + al_live = TRUE; + rc = allocator_recovery_begin((allocator *)&al); + if (!SUCCESS(rc)) { + goto cleanup; + } + if (allocator_in_use((allocator *)&al) != reserved_extent_count) { + platform_error_log("cache_test: rebuild expected %lu reserved extents, " + "found %lu\n", + reserved_extent_count, + allocator_in_use((allocator *)&al)); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + for (uint64 extent_no = 0; extent_no < reserved_extent_count; extent_no++) { + uint64 extent_addr = extent_no * cfg->io_cfg->extent_size; + if (allocator_get_refcount((allocator *)&al, extent_addr) != AL_ONE_REF) { + platform_error_log( + "cache_test: rebuild did not reserve fixed extent %lu\n", + extent_no); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + } + if (allocator_get_refcount((allocator *)&al, stale_extent_addr) != AL_FREE) { + platform_error_log( + "cache_test: rebuild trusted a persisted data refcount\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + // rebuild_acquire establishes the first reference. + rc = allocator_recovery_record_reference( + (allocator *)&al, stale_extent_addr, PAGE_TYPE_MISC); + if (!SUCCESS(rc) + || allocator_get_refcount((allocator *)&al, stale_extent_addr) + != AL_ONE_REF) + { + platform_error_log( + "cache_test: first rebuilt extent reference was incorrect\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + // 4) A failed/abandoned rebuild persists nothing: deinit alone discards it, + // so a later clean open still sees the pre-rebuild map. + rc_allocator_deinit(&al); + al_live = FALSE; + + rc = rc_allocator_mount(&al, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + al_live = TRUE; + rc = allocator_load_refcounts((allocator *)&al); + if (!SUCCESS(rc)) { + goto cleanup; + } + if (allocator_get_refcount((allocator *)&al, stale_extent_addr) + != AL_ONE_REF) + { + platform_error_log( + "cache_test: aborted rebuild leaked into the persisted map\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc_allocator_deinit(&al); + al_live = FALSE; + + // 5) Rebuild + finish + persist round-trips the rebuilt refcount. + rc = rc_allocator_mount(&al, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + al_live = TRUE; + rc = allocator_recovery_begin((allocator *)&al); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = allocator_recovery_record_reference( + (allocator *)&al, stale_extent_addr, PAGE_TYPE_MISC); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = allocator_recovery_record_reference( + (allocator *)&al, stale_extent_addr, PAGE_TYPE_MISC); + if (!SUCCESS(rc) + || allocator_get_refcount((allocator *)&al, stale_extent_addr) + != AL_ONE_REF + 1) + { + platform_error_log( + "cache_test: rebuilt extent reference did not increment\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + allocator_recovery_finish((allocator *)&al); + rc = allocator_persist((allocator *)&al, NULL); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc_allocator_deinit(&al); + al_live = FALSE; + + rc = rc_allocator_mount(&al, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + al_live = TRUE; + rc = allocator_load_refcounts((allocator *)&al); + if (!SUCCESS(rc)) { + goto cleanup; + } + if (allocator_get_refcount((allocator *)&al, stale_extent_addr) + != AL_ONE_REF + 1) + { + platform_error_log("cache_test: finished recovery was not persisted\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + +cleanup: + if (al_live) { + rc_allocator_deinit(&al); + } + + if (SUCCESS(rc)) { + platform_default_log("cache_test: allocator recovery bootstrap test " + "passed\n"); + } + return rc; +} + +#define TEST_MINI_RECOVER_NUM_DATA_EXTENTS 3 + +/* + * Build a real mini_allocator with one metadata extent and several data + * extents, durably persist it, then simulate a crash: mount a fresh allocator + * in recovery mode (which deliberately ignores the persisted refcount table) + * and confirm mini_recover_allocations() rediscovers every extent from the + * on-disk metadata alone -- including the ordering property this whole + * mechanism exists for, since a rebuilding allocator considers all of these + * extents unallocated until mini_recover_allocations() itself establishes + * their references, ahead of the cache_get() calls it issues to read them. + */ +static platform_status +test_mini_recover_allocations(allocator_config *allocator_cfg, + clockcache_config *cache_cfg, + io_handle *io, + platform_heap_id hid) +{ + platform_status rc = STATUS_OK; + rc_allocator original, recovery; + clockcache original_cc, recovery_cc; + bool32 original_al_live = FALSE, original_cc_live = FALSE; + bool32 recovery_al_live = FALSE, recovery_cc_live = FALSE; + mini_allocator mini; + bool32 mini_live = FALSE; + uint64 meta_head = 0; + uint64 data_extents[TEST_MINI_RECOVER_NUM_DATA_EXTENTS] = {0}; + + platform_default_log("cache_test: mini_recover_allocations test started\n"); + + rc = rc_allocator_init( + &original, allocator_cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + original_al_live = TRUE; + + rc = clockcache_init(&original_cc, + cache_cfg, + io, + (allocator *)&original, + "test", + hid, + platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + original_cc_live = TRUE; + cache *original_ccp = (cache *)&original_cc; + + rc = allocator_alloc((allocator *)&original, &meta_head, PAGE_TYPE_MISC); + if (!SUCCESS(rc)) { + goto cleanup; + } + mini_init(&mini, original_ccp, meta_head, 0, 1, PAGE_TYPE_MISC); + mini_live = TRUE; + + for (uint64 i = 0; i < TEST_MINI_RECOVER_NUM_DATA_EXTENTS; i++) { + data_extents[i] = mini_alloc_extent(&mini, 0, NULL); + if (data_extents[i] == 0) { + rc = STATUS_NO_SPACE; + goto cleanup; + } + } + mini_release(&mini); + mini_live = FALSE; + + /* Durably persist before "crashing": recovery must see real disk state. */ + cache_flush(original_ccp); + clockcache_deinit(&original_cc); + original_cc_live = FALSE; + rc = allocator_persist((allocator *)&original, NULL); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc_allocator_deinit(&original); + original_al_live = FALSE; + + rc = rc_allocator_mount( + &recovery, allocator_cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + recovery_al_live = TRUE; + rc = allocator_recovery_begin((allocator *)&recovery); + if (!SUCCESS(rc)) { + goto cleanup; + } + + /* load_refcounts(rebuild) deliberately ignores the persisted map. */ + if (allocator_get_refcount((allocator *)&recovery, meta_head) != AL_FREE) { + platform_error_log( + "cache_test: recovery mount trusted a persisted refcount\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + rc = clockcache_init(&recovery_cc, + cache_cfg, + io, + (allocator *)&recovery, + "test", + hid, + platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + recovery_cc_live = TRUE; + + /* + * Every extent below is still AL_FREE from recovery's point of view: this + * call must establish each one's reference itself, before its own + * cache_get() calls, or those cache_get()s would trip the cache's debug + * allocation check. + */ + rc = mini_recover_allocations( + (cache *)&recovery_cc, meta_head, PAGE_TYPE_MISC); + if (!SUCCESS(rc)) { + platform_error_log("cache_test: mini_recover_allocations failed: %s\n", + platform_status_to_string(rc)); + goto cleanup; + } + + /* + * Exactly one reference per extent is mini_recover_allocations()'s own + * documented contract; reproducing a mini_allocator's full live-lifetime + * refcount (which may include self-references beyond this) is explicitly + * the job of a higher-level recovery walker, not this function. + */ + if (allocator_get_refcount((allocator *)&recovery, meta_head) != AL_ONE_REF) + { + platform_error_log( + "cache_test: mini_recover_allocations did not recover the metadata " + "extent reference\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + for (uint64 i = 0; i < TEST_MINI_RECOVER_NUM_DATA_EXTENTS; i++) { + if (allocator_get_refcount((allocator *)&recovery, data_extents[i]) + != AL_ONE_REF) + { + platform_error_log("cache_test: mini_recover_allocations did not " + "recover data extent %lu\n", + i); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + } + allocator_recovery_finish((allocator *)&recovery); + +cleanup: + if (mini_live) { + mini_release(&mini); + } + if (recovery_cc_live) { + clockcache_deinit(&recovery_cc); + } + if (recovery_al_live) { + rc_allocator_deinit(&recovery); + } + if (original_cc_live) { + clockcache_deinit(&original_cc); + } + if (original_al_live) { + rc_allocator_deinit(&original); + } + + if (SUCCESS(rc)) { + platform_default_log( + "cache_test: mini_recover_allocations test passed\n"); + } + return rc; +} + +typedef struct { + cache *cc; + volatile bool32 finished; + platform_status status; +} cache_fence_test_context; + +static void +cache_test_writeback_fence_thread(void *arg) +{ + cache_fence_test_context *ctxt = (cache_fence_test_context *)arg; + ctxt->status = cache_writeback_dirty(ctxt->cc); + __atomic_store_n(&ctxt->finished, TRUE, __ATOMIC_RELEASE); +} + +/* + * Verify the simplified writeback-fence contract: a single pass flushes every + * dirty, unlocked page and completes even while another page is write-locked, + * skipping the locked page rather than waiting for it. old_page is left dirty + * and unlocked so the fence must flush it; new_page is held write-locked so the + * fence must skip it and still finish promptly. + */ +static platform_status +test_cache_writeback_fence(cache *cc, + clockcache_config *cfg, + platform_heap_id hid) +{ + platform_status rc = STATUS_OK; + uint64 pages_per_extent = cache_config_pages_per_extent(&cfg->super); + uint64 *addr_arr = NULL; + page_handle *old_page = NULL; + page_handle *new_page = NULL; + bool32 new_claimed = FALSE, new_locked = FALSE; + bool32 fence_thread_started = FALSE; + const uint8 old_baseline = 0x11, old_value = 0x22; + const uint8 new_baseline = 0x33, new_value = 0x44; + cache_fence_test_context fence_ctxt = { + .cc = cc, .finished = FALSE, .status = STATUS_OK}; + platform_thread fence_thread; + + platform_default_log("cache_test: writeback fence test started\n"); + platform_assert(cfg->page_capacity >= 2 * pages_per_extent); + + addr_arr = TYPED_ARRAY_MALLOC(hid, addr_arr, 2 * pages_per_extent); + if (addr_arr == NULL) { + rc = STATUS_NO_MEMORY; + goto cleanup; + } + + rc = cache_test_alloc_extents(cc, cfg, addr_arr, 2); + if (!SUCCESS(rc)) { + platform_free(hid, addr_arr); + addr_arr = NULL; + goto cleanup; + } + + rc = cache_test_fill_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), old_baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = cache_test_fill_page(cc, + addr_arr[pages_per_extent], + cache_config_page_size(&cfg->super), + new_baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + cache_flush(cc); + + /* Dirty old_page and leave it unlocked: the fence must flush it. */ + old_page = cache_get(cc, addr_arr[0], TRUE, PAGE_TYPE_MISC); + if (!cache_try_claim(cc, old_page)) { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + cache_lock(cc, old_page); + memset(old_page->data, old_value, cache_config_page_size(&cfg->super)); + cache_unlock(cc, old_page); + cache_unclaim(cc, old_page); + cache_unget(cc, old_page); + old_page = NULL; + + /* Hold new_page write-locked: the fence must skip it and still complete. */ + new_page = cache_get(cc, addr_arr[pages_per_extent], TRUE, PAGE_TYPE_MISC); + if (!cache_try_claim(cc, new_page)) { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + new_claimed = TRUE; + cache_lock(cc, new_page); + new_locked = TRUE; + memset(new_page->data, new_value, cache_config_page_size(&cfg->super)); + + rc = platform_thread_create(&fence_thread, + FALSE, + cache_test_writeback_fence_thread, + &fence_ctxt, + hid); + if (!SUCCESS(rc)) { + goto cleanup; + } + fence_thread_started = TRUE; + + /* The fence must finish without waiting for the write-locked new_page. */ + timestamp wait_start = platform_get_timestamp(); + while (!__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { + if (platform_timestamp_elapsed(wait_start) > SEC_TO_NSEC(5)) { + platform_error_log( + "cache_test: fence did not complete while a page was locked\n"); + rc = STATUS_TIMEDOUT; + goto cleanup; + } + platform_sleep_ns(1000); + } + +cleanup: + if (new_locked) { + cache_unlock(cc, new_page); + } + if (new_claimed) { + cache_unclaim(cc, new_page); + } + if (new_page != NULL) { + cache_unget(cc, new_page); + } + if (old_page != NULL) { + cache_unget(cc, old_page); + } + if (fence_thread_started) { + platform_status join_rc = platform_thread_join(&fence_thread); + if (!SUCCESS(join_rc) && SUCCESS(rc)) { + rc = join_rc; + } + if (!SUCCESS(fence_ctxt.status) && SUCCESS(rc)) { + rc = fence_ctxt.status; + } + } + + /* Only new_page, write-locked throughout the fence, remains dirty. */ + if (SUCCESS(rc)) { + uint32 dirty_count = cache_count_dirty(cc); + if (dirty_count != 1) { + platform_error_log("cache_test: expected one dirty page after fence, " + "found %u\n", + dirty_count); + rc = STATUS_TEST_FAILED; + } + } + if (SUCCESS(rc)) { + rc = cache_durable_barrier(cc); + } + /* The dirty, unlocked page was flushed by the fence. */ + if (SUCCESS(rc)) { + rc = cache_test_verify_disk_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), old_value); + } + /* The write-locked page was skipped, so its baseline is still on disk. */ + if (SUCCESS(rc)) { + rc = cache_test_verify_disk_page(cc, + addr_arr[pages_per_extent], + cache_config_page_size(&cfg->super), + new_baseline); + } + + if (addr_arr != NULL) { + cache_flush(cc); + for (uint32 i = 0; i < 2; i++) { + uint64 addr = addr_arr[i * pages_per_extent]; + allocator *al = cache_get_allocator(cc); + refcount ref = allocator_dec_ref(al, addr, PAGE_TYPE_MISC); + platform_assert(ref == AL_NO_REFS); + cache_extent_discard(cc, addr, PAGE_TYPE_MISC); + ref = allocator_dec_ref(al, addr, PAGE_TYPE_MISC); + platform_assert(ref == AL_FREE); + } + platform_free(hid, addr_arr); + } + + if (SUCCESS(rc)) { + platform_default_log("cache_test: writeback fence test passed\n"); + } else { + platform_default_log("cache_test: writeback fence test failed\n"); + } + return rc; +} + +/* + * A page that is only claimed (not write-locked) is not cleanable, so a fence + * skips it and completes; the page's baseline is therefore still what is on + * disk. Once the claim is dropped, a second fence flushes the page. + */ +static platform_status +test_cache_writeback_fence_claimed_page(cache *cc, + clockcache_config *cfg, + platform_heap_id hid) +{ + platform_status rc = STATUS_OK; + uint64 pages_per_extent = cache_config_pages_per_extent(&cfg->super); + uint64 *addr_arr = NULL; + page_handle *page = NULL; + bool32 claimed = FALSE, locked = FALSE; + bool32 fence_thread_started = FALSE; + const uint8 baseline = 0x55, final_value = 0x66; + cache_fence_test_context fence_ctxt = { + .cc = cc, .finished = FALSE, .status = STATUS_OK}; + platform_thread fence_thread; + + platform_default_log("cache_test: claimed-page fence test started\n"); + + addr_arr = TYPED_ARRAY_MALLOC(hid, addr_arr, pages_per_extent); + if (addr_arr == NULL) { + rc = STATUS_NO_MEMORY; + goto cleanup; + } + + rc = cache_test_alloc_extents(cc, cfg, addr_arr, 1); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = cache_test_fill_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + cache_flush(cc); + + /* Dirty the page, then keep only a claim (release the write lock). */ + page = cache_get(cc, addr_arr[0], TRUE, PAGE_TYPE_MISC); + if (!cache_try_claim(cc, page)) { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + claimed = TRUE; + cache_lock(cc, page); + locked = TRUE; + memset(page->data, final_value, cache_config_page_size(&cfg->super)); + cache_unlock(cc, page); + locked = FALSE; + + rc = platform_thread_create(&fence_thread, + FALSE, + cache_test_writeback_fence_thread, + &fence_ctxt, + hid); + if (!SUCCESS(rc)) { + goto cleanup; + } + fence_thread_started = TRUE; + + /* A claimed page is not cleanable, so the fence skips it and completes. */ + timestamp wait_start = platform_get_timestamp(); + while (!__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { + if (platform_timestamp_elapsed(wait_start) > SEC_TO_NSEC(5)) { + platform_error_log( + "cache_test: fence did not complete while page was claimed\n"); + rc = STATUS_TIMEDOUT; + goto cleanup; + } + platform_sleep_ns(1000); + } + + platform_status join_rc = platform_thread_join(&fence_thread); + fence_thread_started = FALSE; + if (!SUCCESS(join_rc)) { + rc = join_rc; + goto cleanup; + } + if (!SUCCESS(fence_ctxt.status)) { + rc = fence_ctxt.status; + goto cleanup; + } + + /* The skipped page was not written, so its baseline is still on disk. */ + rc = cache_test_verify_disk_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + + /* Drop the claim; the page is now cleanable and a second fence flushes it. + */ + cache_unclaim(cc, page); + claimed = FALSE; + cache_unget(cc, page); + page = NULL; + + rc = cache_writeback_dirty(cc); + if (!SUCCESS(rc)) { + goto cleanup; + } + if (cache_count_dirty(cc) != 0) { + platform_error_log("cache_test: second fence left dirty pages\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc = cache_durable_barrier(cc); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = cache_test_verify_disk_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), final_value); + +cleanup: + if (locked) { + cache_unlock(cc, page); + } + if (claimed) { + cache_unclaim(cc, page); + } + if (page != NULL) { + cache_unget(cc, page); + } + if (fence_thread_started) { + platform_status jrc = platform_thread_join(&fence_thread); + if (!SUCCESS(jrc) && SUCCESS(rc)) { + rc = jrc; + } + if (!SUCCESS(fence_ctxt.status) && SUCCESS(rc)) { + rc = fence_ctxt.status; + } + } + + if (addr_arr != NULL) { + cache_flush(cc); + allocator *al = cache_get_allocator(cc); + refcount ref = allocator_dec_ref(al, addr_arr[0], PAGE_TYPE_MISC); + platform_assert(ref == AL_NO_REFS); + cache_extent_discard(cc, addr_arr[0], PAGE_TYPE_MISC); + ref = allocator_dec_ref(al, addr_arr[0], PAGE_TYPE_MISC); + platform_assert(ref == AL_FREE); + platform_free(hid, addr_arr); + } + + if (SUCCESS(rc)) { + platform_default_log("cache_test: claimed-page fence test passed\n"); + } else { + platform_default_log("cache_test: claimed-page fence test failed\n"); + } + return rc; +} + +typedef struct { + cache *cc; + uint64 *addrs; + uint64 num_addrs; + uint64 page_size; + volatile bool32 stop; +} cache_hammer_context; + +/* + * Continuously dirty and asynchronously write back a rotating set of pages, so + * the I/O subsystem never reaches global quiescence. + */ +static void +cache_test_hammer_thread(void *arg) +{ + cache_hammer_context *ctxt = (cache_hammer_context *)arg; + uint64 i = 0; + while (!__atomic_load_n(&ctxt->stop, __ATOMIC_ACQUIRE)) { + uint64 addr = ctxt->addrs[i % ctxt->num_addrs]; + page_handle *page = cache_get(ctxt->cc, addr, TRUE, PAGE_TYPE_MISC); + if (cache_try_claim(ctxt->cc, page)) { + cache_lock(ctxt->cc, page); + memset(page->data, (uint8)i, ctxt->page_size); + cache_unlock(ctxt->cc, page); + cache_unclaim(ctxt->cc, page); + cache_page_writeback(ctxt->cc, page, FALSE, PAGE_TYPE_MISC); + } + cache_unget(ctxt->cc, page); + i++; + } +} + +/* + * A writeback fence must drain only the pages dirty at its cut and then return, + * even while another thread keeps dirtying and writing back other pages. The + * previous io_wait_all implementation waited for *global* I/O quiescence, which + * this workload never reaches, so it would hang here. + */ +static platform_status +test_cache_writeback_fence_liveness(cache *cc, + clockcache_config *cfg, + platform_heap_id hid) +{ + platform_status rc = STATUS_OK; + uint64 page_size = cache_config_page_size(&cfg->super); + uint64 pages_per_extent = cache_config_pages_per_extent(&cfg->super); + uint64 *addr_arr = NULL; + const uint8 baseline = 0x11, target_value = 0x77; + bool32 hammer_started = FALSE, fence_started = FALSE; + cache_hammer_context hammer_ctxt = {.cc = cc, .stop = FALSE}; + cache_fence_test_context fence_ctxt = { + .cc = cc, .finished = FALSE, .status = STATUS_OK}; + platform_thread hammer_thread, fence_thread; + + platform_default_log("cache_test: writeback fence liveness test started\n"); + platform_assert(cfg->page_capacity >= 2 * pages_per_extent); + + addr_arr = TYPED_ARRAY_MALLOC(hid, addr_arr, 2 * pages_per_extent); + if (addr_arr == NULL) { + rc = STATUS_NO_MEMORY; + goto cleanup; + } + rc = cache_test_alloc_extents(cc, cfg, addr_arr, 2); + if (!SUCCESS(rc)) { + platform_free(hid, addr_arr); + addr_arr = NULL; + goto cleanup; + } + rc = cache_test_fill_page(cc, addr_arr[0], page_size, baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + cache_flush(cc); + + /* Dirty the target page (unlocked); the fence must flush it. */ + rc = cache_test_fill_page(cc, addr_arr[0], page_size, target_value); + if (!SUCCESS(rc)) { + goto cleanup; + } + + /* Hammer the second extent's pages to keep I/O perpetually in flight. */ + hammer_ctxt.addrs = &addr_arr[pages_per_extent]; + hammer_ctxt.num_addrs = pages_per_extent; + hammer_ctxt.page_size = page_size; + rc = platform_thread_create( + &hammer_thread, FALSE, cache_test_hammer_thread, &hammer_ctxt, hid); + if (!SUCCESS(rc)) { + goto cleanup; + } + hammer_started = TRUE; + + rc = platform_thread_create(&fence_thread, + FALSE, + cache_test_writeback_fence_thread, + &fence_ctxt, + hid); + if (!SUCCESS(rc)) { + goto cleanup; + } + fence_started = TRUE; + + /* The fence must return despite the ongoing hammer activity. */ + timestamp wait_start = platform_get_timestamp(); + while (!__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { + if (platform_timestamp_elapsed(wait_start) > SEC_TO_NSEC(10)) { + platform_error_log( + "cache_test: fence did not terminate under concurrent writes\n"); + rc = STATUS_TIMEDOUT; + break; + } + platform_sleep_ns(1000); + } + +cleanup: + /* Stop the hammer first so I/O can quiesce even if the fence is stuck. */ + if (hammer_started) { + __atomic_store_n(&hammer_ctxt.stop, TRUE, __ATOMIC_RELEASE); + platform_status jrc = platform_thread_join(&hammer_thread); + if (!SUCCESS(jrc) && SUCCESS(rc)) { + rc = jrc; + } + } + if (fence_started) { + platform_status jrc = platform_thread_join(&fence_thread); + if (!SUCCESS(jrc) && SUCCESS(rc)) { + rc = jrc; + } + if (!SUCCESS(fence_ctxt.status) && SUCCESS(rc)) { + rc = fence_ctxt.status; + } + } + + /* The target page, dirty at the cut, must have been made durable. */ + if (SUCCESS(rc)) { + rc = cache_durable_barrier(cc); + } + if (SUCCESS(rc)) { + rc = + cache_test_verify_disk_page(cc, addr_arr[0], page_size, target_value); + } + + if (addr_arr != NULL) { + cache_flush(cc); + for (uint32 i = 0; i < 2; i++) { + uint64 addr = addr_arr[i * pages_per_extent]; + allocator *al = cache_get_allocator(cc); + refcount ref = allocator_dec_ref(al, addr, PAGE_TYPE_MISC); + platform_assert(ref == AL_NO_REFS); + cache_extent_discard(cc, addr, PAGE_TYPE_MISC); + ref = allocator_dec_ref(al, addr, PAGE_TYPE_MISC); + platform_assert(ref == AL_FREE); + } + platform_free(hid, addr_arr); + } + + if (SUCCESS(rc)) { + platform_default_log( + "cache_test: writeback fence liveness test passed\n"); + } else { + platform_default_log( + "cache_test: writeback fence liveness test failed\n"); + } + return rc; +} + platform_status test_cache_basic(cache *cc, clockcache_config *cfg, platform_heap_id hid) { @@ -98,6 +1040,21 @@ test_cache_basic(cache *cc, clockcache_config *cfg, platform_heap_id hid) page_handle **page_arr = NULL; uint64 *addr_arr = NULL; + rc = test_cache_writeback_fence(cc, cfg, hid); + if (!SUCCESS(rc)) { + goto exit; + } + + rc = test_cache_writeback_fence_claimed_page(cc, cfg, hid); + if (!SUCCESS(rc)) { + goto exit; + } + + rc = test_cache_writeback_fence_liveness(cc, cfg, hid); + if (!SUCCESS(rc)) { + goto exit; + } + /* allocate twice as many pages as the cache capacity */ uint64 pages_per_extent = cache_config_pages_per_extent(&cfg->super); uint32 extent_capacity = cfg->page_capacity / pages_per_extent; @@ -232,7 +1189,6 @@ test_cache_basic(cache *cc, clockcache_config *cfg, platform_heap_id hid) } } for (i = 0; i < cfg->page_capacity; i++) { - cache_mark_dirty(cc, page_arr[i]); cache_unlock(cc, page_arr[i]); cache_unclaim(cc, page_arr[i]); cache_unget(cc, page_arr[i]); @@ -418,7 +1374,6 @@ cache_test_dirty_flush(cache *cc, rc = STATUS_TEST_FAILED; break; } - cache_mark_dirty(cc, ph); cache_unlock(cc, ph); cache_unclaim(cc, ph); cache_unget(cc, ph); @@ -977,6 +1932,18 @@ cache_test(int argc, char *argv[]) goto cleanup; } + rc = + test_rc_allocator_recovery_bootstrap(&system_cfg.allocator_cfg, io, hid); + if (!SUCCESS(rc)) { + goto destroy_iohandle; + } + + rc = test_mini_recover_allocations( + &system_cfg.allocator_cfg, &system_cfg.cache_cfg, io, hid); + if (!SUCCESS(rc)) { + goto destroy_iohandle; + } + rc = test_init_task_system(&ts, hid, &system_cfg.task_cfg); if (!SUCCESS(rc)) { platform_error_log("Failed to init splinter state: %s\n", diff --git a/tests/functional/log_test.c b/tests/functional/log_test.c index 1bf96ba5..2cdcb39d 100644 --- a/tests/functional/log_test.c +++ b/tests/functional/log_test.c @@ -19,13 +19,14 @@ #include "poison.h" +#define LOG_TEST_LEAVES_PER_MEMTABLE 97 + int test_log_crash(clockcache *cc, clockcache_config *cache_cfg, io_handle *io, allocator *al, shard_log_config *cfg, - shard_log *log, task_system *ts, platform_heap_id hid, test_message_generator *gen, @@ -34,37 +35,66 @@ test_log_crash(clockcache *cc, bool32 crash) { - platform_status rc; - log_handle *logh; - uint64 i; - key returned_key; - message returned_message; - uint64 addr; - uint64 magic; - shard_log_iterator itor; - iterator *itorh = (iterator *)&itor; - char key_str[128]; - char data_str[128]; - merge_accumulator msg; + platform_status rc; + log_handle *logh; + uint64 i; + key returned_key; + message returned_message; + log_head segment; + log_iterator *itor; + char key_str[128]; + char data_str[128]; + merge_accumulator msg; DECLARE_AUTO_KEY_BUFFER(keybuffer, hid); platform_assert(cc != NULL); - rc = shard_log_init(log, (cache *)cc, cfg); - platform_assert_status_ok(rc); - logh = (log_handle *)log; + logh = shard_log_create((cache *)cc, cfg, hid); + platform_assert(logh != NULL); - addr = log_addr(logh); - magic = log_magic(logh); + // The identity is fixed at creation; capture it before writing/sealing. + segment = log_get_head(logh); merge_accumulator_init(&msg, hid); for (i = 0; i < num_entries; i++) { - key skey = - test_key(&keybuffer, TEST_RANDOM, i, 0, 0, 1 + (i % key_size), 0); - generate_test_message(gen, i, &msg); - log_write(logh, skey, merge_accumulator_to_message(&msg), i); + uint64 entry_num = i; + if (2 * LOG_TEST_LEAVES_PER_MEMTABLE <= num_entries + && i < 2 * LOG_TEST_LEAVES_PER_MEMTABLE) + { + /* + * Write the first two generations in the opposite order, and each + * generation in reverse leaf order. The iterator must restore the + * (memtable_generation, leaf_generation) order below. + */ + uint64 memtable_generation = 1 - i / LOG_TEST_LEAVES_PER_MEMTABLE; + uint64 leaf_generation = + LOG_TEST_LEAVES_PER_MEMTABLE - 1 - i % LOG_TEST_LEAVES_PER_MEMTABLE; + entry_num = memtable_generation * LOG_TEST_LEAVES_PER_MEMTABLE + + leaf_generation; + } + key skey = test_key(&keybuffer, + TEST_RANDOM, + entry_num, + 0, + 0, + 1 + (entry_num % key_size), + 0); + generate_test_message(gen, entry_num, &msg); + int log_rc = log_write(logh, + skey, + merge_accumulator_to_message(&msg), + entry_num / LOG_TEST_LEAVES_PER_MEMTABLE, + entry_num % LOG_TEST_LEAVES_PER_MEMTABLE); + platform_assert(log_rc == 0); } + rc = log_seal(logh); // frees logh; identity captured above + platform_assert_status_ok(rc); + rc = cache_writeback_dirty((cache *)cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier((cache *)cc); + platform_assert_status_ok(rc); + if (crash) { clockcache_deinit(cc); rc = clockcache_init( @@ -72,16 +102,21 @@ test_log_crash(clockcache *cc, platform_assert_status_ok(rc); } - rc = shard_log_iterator_init((cache *)cc, cfg, hid, addr, magic, &itor); - platform_assert_status_ok(rc); - itorh = (iterator *)&itor; + itor = shard_log_iterator_create((cache *)cc, cfg, hid, segment); + platform_assert(itor != NULL); - for (i = 0; i < num_entries && iterator_can_curr(itorh); i++) { + for (i = 0; i < num_entries && log_iterator_can_next(itor); i++) { key skey = test_key(&keybuffer, TEST_RANDOM, i, 0, 0, 1 + (i % key_size), 0); generate_test_message(gen, i, &msg); message mmessage = merge_accumulator_to_message(&msg); - iterator_curr(itorh, &returned_key, &returned_message); + log_iterator_curr(itor, &returned_key, &returned_message); + uint64 memtable_generation; + uint64 leaf_generation; + log_iterator_curr_generations( + itor, &memtable_generation, &leaf_generation); + platform_assert(memtable_generation == i / LOG_TEST_LEAVES_PER_MEMTABLE); + platform_assert(leaf_generation == i % LOG_TEST_LEAVES_PER_MEMTABLE); if (data_key_compare(cfg->data_cfg, skey, returned_key) || message_lex_cmp(mmessage, returned_message)) { @@ -94,47 +129,202 @@ test_log_crash(clockcache *cc, platform_default_log("actual: %s -- %s\n", key_str, data_str); platform_assert(0); } - rc = iterator_next(itorh); + rc = log_iterator_next(itor); platform_assert_status_ok(rc); } platform_default_log("log returned %lu of %lu entries\n", i, num_entries); + platform_assert(i == num_entries); + platform_assert(!log_iterator_can_next(itor)); merge_accumulator_deinit(&msg); - shard_log_iterator_deinit(hid, &itor); - shard_log_zap(log); + log_iterator_deinit(itor); + log_dec_ref((cache *)cc, &segment); return 0; } +static void +test_log_write_range(log_handle *logh, + test_message_generator *gen, + platform_heap_id hid, + uint64 key_size, + uint64 first_entry, + uint64 num_entries) +{ + merge_accumulator msg; + DECLARE_AUTO_KEY_BUFFER(keybuffer, hid); + merge_accumulator_init(&msg, hid); + + for (uint64 i = 0; i < num_entries; i++) { + uint64 entry_num = first_entry + i; + key skey = test_key(&keybuffer, + TEST_RANDOM, + entry_num, + 0, + 0, + 1 + (entry_num % key_size), + 0); + generate_test_message(gen, entry_num, &msg); + int log_rc = log_write( + logh, skey, merge_accumulator_to_message(&msg), entry_num, 0); + platform_assert(log_rc == 0); + } + + merge_accumulator_deinit(&msg); +} + +static void +test_log_verify_segment(cache *cc, + shard_log_config *cfg, + const log_head *segment, + test_message_generator *gen, + platform_heap_id hid, + uint64 key_size, + uint64 first_entry, + uint64 num_entries) +{ + log_iterator *itor; + merge_accumulator msg; + DECLARE_AUTO_KEY_BUFFER(keybuffer, hid); + key returned_key; + message returned_message; + + platform_assert(segment->addr != 0); + platform_assert(segment->meta_addr != 0); + itor = shard_log_iterator_create(cc, cfg, hid, *segment); + platform_assert(itor != NULL); + + merge_accumulator_init(&msg, hid); + for (uint64 i = 0; i < num_entries; i++) { + uint64 entry_num = first_entry + i; + platform_assert(log_iterator_can_next(itor)); + key skey = test_key(&keybuffer, + TEST_RANDOM, + entry_num, + 0, + 0, + 1 + (entry_num % key_size), + 0); + generate_test_message(gen, entry_num, &msg); + log_iterator_curr(itor, &returned_key, &returned_message); + uint64 memtable_generation; + uint64 leaf_generation; + log_iterator_curr_generations( + itor, &memtable_generation, &leaf_generation); + platform_assert(memtable_generation == entry_num); + platform_assert(leaf_generation == 0); + platform_assert(data_key_compare(cfg->data_cfg, skey, returned_key) == 0); + platform_assert( + message_lex_cmp(merge_accumulator_to_message(&msg), returned_message) + == 0); + platform_status rc = log_iterator_next(itor); + platform_assert_status_ok(rc); + } + platform_assert(!log_iterator_can_next(itor)); + + merge_accumulator_deinit(&msg); + log_iterator_deinit(itor); +} + +/* + * Sealing a stream and creating a fresh one must yield two distinct, + * independently replayable segments. Reinitializing the cache after each forced + * physical persistence cut makes this test exercise only persisted pages for + * both identities; it does not model logical durable-tail publication. + */ static int -test_log_large_message(cache *cc, - shard_log_config *cfg, - shard_log *log, - platform_heap_id hid) +test_log_two_segments(clockcache *cc, + clockcache_config *cache_cfg, + io_handle *io, + allocator *al, + shard_log_config *cfg, + platform_heap_id hid, + test_message_generator *gen, + uint64 key_size) { - platform_status rc; - shard_log_iterator itor; - iterator *itorh = (iterator *)&itor; - merge_accumulator msg; - key returned_key; - message returned_message; - char key_data[] = "large-log-key"; - key skey = key_create(FALSE, sizeof(key_data) - 1, key_data); - uint64 value_len = 3 * cache_page_size(cc) + 123; - - rc = shard_log_init(log, cc, cfg); + const uint64 old_first = 1000, old_count = 16; + const uint64 new_first = 2000, new_count = 16; + log_head sealed, fresh; + + log_handle *log = shard_log_create((cache *)cc, cfg, hid); + platform_assert(log != NULL); + sealed = log_get_head(log); // identity is fixed at creation + test_log_write_range(log, gen, hid, key_size, old_first, old_count); + + platform_status rc = log_seal(log); // frees log + platform_assert_status_ok(rc); + platform_assert(sealed.addr != 0); + platform_assert(sealed.meta_addr != 0); + + rc = cache_writeback_dirty((cache *)cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier((cache *)cc); + platform_assert_status_ok(rc); + + clockcache_deinit(cc); + rc = clockcache_init( + cc, cache_cfg, io, al, "sealed-old", hid, platform_get_module_id()); + platform_assert_status_ok(rc); + test_log_verify_segment( + (cache *)cc, cfg, &sealed, gen, hid, key_size, old_first, old_count); + + // A fresh stream is a distinct segment: new mini allocator and new magic. + log = shard_log_create((cache *)cc, cfg, hid); + platform_assert(log != NULL); + fresh = log_get_head(log); + test_log_write_range(log, gen, hid, key_size, new_first, new_count); + rc = log_seal(log); // frees log + platform_assert_status_ok(rc); + platform_assert(fresh.addr != 0); + platform_assert(fresh.meta_addr != 0); + platform_assert(sealed.meta_addr != fresh.meta_addr); + platform_assert(sealed.magic != fresh.magic); + + rc = cache_writeback_dirty((cache *)cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier((cache *)cc); platform_assert_status_ok(rc); + clockcache_deinit(cc); + rc = clockcache_init( + cc, cache_cfg, io, al, "sealed-new", hid, platform_get_module_id()); + platform_assert_status_ok(rc); + test_log_verify_segment( + (cache *)cc, cfg, &sealed, gen, hid, key_size, old_first, old_count); + test_log_verify_segment( + (cache *)cc, cfg, &fresh, gen, hid, key_size, new_first, new_count); + + log_dec_ref((cache *)cc, &sealed); + log_dec_ref((cache *)cc, &fresh); + return 0; +} + +static int +test_log_large_message(cache *cc, shard_log_config *cfg, platform_heap_id hid) +{ + platform_status rc; + log_head sealed; + log_iterator *itor; + merge_accumulator msg; + key returned_key; + message returned_message; + char key_data[] = "large-log-key"; + key skey = key_create(FALSE, sizeof(key_data) - 1, key_data); + uint64 value_len = 3 * cache_page_size(cc) + 123; + + log_handle *logh = shard_log_create(cc, cfg, hid); + platform_assert(logh != NULL); + sealed = log_get_head(logh); // identity is fixed at creation + merge_accumulator_init(&msg, hid); bool32 success = merge_accumulator_resize(&msg, value_len); platform_assert(success); merge_accumulator_set_class(&msg, MESSAGE_TYPE_INSERT); memset(merge_accumulator_data(&msg), 'L', value_len); - int log_rc = - log_write((log_handle *)log, skey, merge_accumulator_to_message(&msg), 0); + int log_rc = log_write(logh, skey, merge_accumulator_to_message(&msg), 0, 0); platform_assert(log_rc == 0); merge_accumulator filler; @@ -146,34 +336,36 @@ test_log_large_message(cache *cc, merge_accumulator_data(&filler), 'f', merge_accumulator_length(&filler)); for (uint64 i = 1; i < 16; i++) { log_rc = log_write( - (log_handle *)log, skey, merge_accumulator_to_message(&filler), i); + logh, skey, merge_accumulator_to_message(&filler), i / 4, i % 4); platform_assert(log_rc == 0); } merge_accumulator_deinit(&filler); - rc = shard_log_iterator_init(cc, - cfg, - hid, - log_addr((log_handle *)log), - log_magic((log_handle *)log), - &itor); + rc = log_seal(logh); // frees logh; identity captured above + platform_assert_status_ok(rc); + rc = cache_writeback_dirty(cc); platform_assert_status_ok(rc); - platform_assert(iterator_can_curr(itorh)); + rc = cache_durable_barrier(cc); + platform_assert_status_ok(rc); + + itor = shard_log_iterator_create(cc, cfg, hid, sealed); + platform_assert(itor != NULL); + platform_assert(log_iterator_can_next(itor)); - iterator_curr(itorh, &returned_key, &returned_message); + log_iterator_curr(itor, &returned_key, &returned_message); platform_assert(data_key_compare(cfg->data_cfg, skey, returned_key) == 0); platform_assert( message_lex_cmp(merge_accumulator_to_message(&msg), returned_message) == 0); - shard_log_iterator_deinit(hid, &itor); + log_iterator_deinit(itor); merge_accumulator_deinit(&msg); - shard_log_zap(log); + log_dec_ref(cc, &sealed); return 0; } typedef struct test_log_thread_params { - shard_log *log; + log_handle *logh; platform_thread thread; int thread_id; test_message_generator *gen; @@ -186,8 +378,7 @@ test_log_thread(void *arg) { platform_heap_id hid = platform_get_heap_id(); test_log_thread_params *params = (test_log_thread_params *)arg; - shard_log *log = params->log; - log_handle *logh = (log_handle *)log; + log_handle *logh = params->logh; int thread_id = params->thread_id; uint64 num_entries = params->num_entries; test_message_generator *gen = params->gen; @@ -201,7 +392,9 @@ test_log_thread(void *arg) for (i = thread_id * num_entries; i < (thread_id + 1) * num_entries; i++) { key skey = test_key(&keybuf, TEST_RANDOM, i, 0, 0, key_size, 0); generate_test_message(gen, i, &msg); - log_write(logh, skey, merge_accumulator_to_message(&msg), i); + int log_rc = log_write( + logh, skey, merge_accumulator_to_message(&msg), i / 1024, i % 1024); + platform_assert(log_rc == 0); } merge_accumulator_deinit(&msg); @@ -210,7 +403,6 @@ test_log_thread(void *arg) platform_status test_log_perf(cache *cc, shard_log_config *cfg, - shard_log *log, uint64 num_entries, test_message_generator *gen, uint64 key_size, @@ -225,11 +417,12 @@ test_log_perf(cache *cc, uint64 start_time; platform_status ret; - ret = shard_log_init(log, (cache *)cc, cfg); - platform_assert_status_ok(ret); + log_handle *logh = shard_log_create((cache *)cc, cfg, hid); + platform_assert(logh != NULL); + log_head sealed = log_get_head(logh); for (uint64 i = 0; i < num_threads; i++) { - params[i].log = log; + params[i].logh = logh; params[i].thread_id = i; params[i].gen = gen; params[i].key_size = key_size; @@ -257,6 +450,9 @@ test_log_perf(cache *cc, / platform_timestamp_elapsed(start_time)); cleanup: + // Seal (frees the handle) and release the segment's extents. + log_seal(logh); + log_dec_ref((cache *)cc, &sealed); platform_free(hid, params); return ret; @@ -375,15 +571,22 @@ log_test(int argc, char *argv[]) platform_get_module_id()); platform_assert_status_ok(status); - shard_log *log = TYPED_MALLOC(hid, log); - platform_assert(log != NULL); - rc = test_log_large_message((cache *)cc, &system_cfg.log_cfg, log, hid); + rc = test_log_large_message((cache *)cc, &system_cfg.log_cfg, hid); + platform_assert(rc == 0); + + rc = test_log_two_segments(cc, + &system_cfg.cache_cfg, + io, + (allocator *)&al, + &system_cfg.log_cfg, + hid, + &gen, + workload_cfg.key_size); platform_assert(rc == 0); if (run_perf_test) { ret = test_log_perf((cache *)cc, &system_cfg.log_cfg, - log, 200000000, &gen, workload_cfg.key_size, @@ -398,7 +601,6 @@ log_test(int argc, char *argv[]) io, (allocator *)&al, &system_cfg.log_cfg, - log, &ts, hid, &gen, @@ -412,7 +614,6 @@ log_test(int argc, char *argv[]) io, (allocator *)&al, &system_cfg.log_cfg, - log, &ts, hid, &gen, @@ -424,7 +625,6 @@ log_test(int argc, char *argv[]) io_wait_all(io); clockcache_deinit(cc); - platform_free(hid, log); platform_free(hid, cc); rc_allocator_deinit(&al); test_deinit_task_system(&ts); diff --git a/tests/functional/splinter_test.c b/tests/functional/splinter_test.c index 5e665647..6671adb3 100644 --- a/tests/functional/splinter_test.c +++ b/tests/functional/splinter_test.c @@ -51,10 +51,9 @@ typedef struct test_splinter_thread_params { platform_thread thread; core_handle *spl; test_config *test_cfg; - uint64 *total_ops; + uint64 total_ops; uint64 *curr_op; uint64 op_granularity; - uint8 num_tables; uint64 thread_number; task_system *ts; platform_status rc; @@ -63,7 +62,7 @@ typedef struct test_splinter_thread_params { stats_insert insert_stats; uint64 num_ops_per_thread[NUM_OP_TYPES]; // in each round bool32 expected_found; - test_async_lookup *async_lookup[8]; // async lookup state per table + test_async_lookup *async_lookup; uint64 insert_rate; stats_lookup lookup_stats[NUM_LOOKUP_TYPES]; uint8 lookup_positive_pct; // parallel lookup positive % @@ -91,24 +90,6 @@ typedef struct trunk_range_perf_params { */ typedef void (*test_trunk_thread_hdlr)(void *arg); -static inline bool32 -test_is_done(const uint8 done, const uint8 n) -{ - return (((done >> n) & 1) != 0); -} - -static inline void -test_set_done(uint8 *done, const uint8 n) -{ - *done |= 1 << n; -} - -static inline bool32 -test_all_done(const uint8 done, const uint8 num_tables) -{ - return (done == ((1 << num_tables) - 1)); -} - /* * test_trunk_insert_thread() -- Per-thread function to drive inserts. */ @@ -117,17 +98,13 @@ test_trunk_insert_thread(void *arg) { test_splinter_thread_params *params = (test_splinter_thread_params *)arg; - core_handle *spl_tables = params->spl; + core_handle *spl = params->spl; const test_config *test_cfg = params->test_cfg; - const uint64 *total_ops = params->total_ops; + const uint64 total_ops = params->total_ops; uint64 *curr_op = params->curr_op; uint64 op_granularity = params->op_granularity; uint64 thread_number = params->thread_number; - uint8 num_tables = params->num_tables; platform_heap_id heap_id = platform_get_heap_id(); - platform_assert(num_tables <= 8); - uint64 *insert_base = TYPED_ARRAY_ZALLOC(heap_id, insert_base, num_tables); - uint8 done = 0; uint64 num_inserts = 0; timestamp next_check_time = platform_get_timestamp(); @@ -136,76 +113,60 @@ test_trunk_insert_thread(void *arg) merge_accumulator msg; merge_accumulator_init(&msg, heap_id); + uint64 insert_base = 0; while (1) { - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - if (test_is_done(done, spl_idx)) { - continue; - } - test_print_progress(¶ms->progress, - insert_base[spl_idx] / (total_ops[spl_idx] / 100), - PLATFORM_CR "Thread %lu inserting %3lu%% " - "complete for table %u", - thread_number, - insert_base[spl_idx] / (total_ops[spl_idx] / 100), - spl_idx); - insert_base[spl_idx] = - __sync_fetch_and_add(&curr_op[spl_idx], op_granularity); - if (insert_base[spl_idx] >= total_ops[spl_idx]) { - test_set_done(&done, spl_idx); - } - if (test_all_done(done, num_tables)) { - platform_default_log(" Test done for all %d tables.\n", num_tables); - goto out; - } + test_print_progress(¶ms->progress, + insert_base / (total_ops / 100), + PLATFORM_CR "Thread %lu inserting %3lu%% complete", + thread_number, + insert_base / (total_ops / 100)); + insert_base = __sync_fetch_and_add(curr_op, op_granularity); + if (insert_base >= total_ops) { + platform_default_log(" Test done.\n"); + goto out; } DECLARE_AUTO_KEY_BUFFER(keybuf, heap_id); for (uint64 op_offset = 0; op_offset != op_granularity; op_offset++) { - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - uint64 insert_num = insert_base[spl_idx] + op_offset; - if (test_is_done(done, spl_idx)) { - continue; - } - core_handle *spl = &spl_tables[spl_idx]; + uint64 insert_num = insert_base + op_offset; - timestamp ts; - if (spl->cfg.use_stats) { - ts = platform_get_timestamp(); - } - test_key(&keybuf, - test_cfg[spl_idx].key_type, - insert_num, - thread_number, - test_cfg[spl_idx].semiseq_freq, - test_cfg[spl_idx].key_size, - test_cfg[spl_idx].period); - generate_test_message(test_cfg->gen, insert_num, &msg); - platform_status rc = core_insert(spl, - key_buffer_key(&keybuf), - merge_accumulator_to_message(&msg), - NULL); - platform_assert_status_ok(rc); - if (spl->cfg.use_stats) { - ts = platform_timestamp_elapsed(ts); - if (ts > params->insert_stats.latency_max) { - params->insert_stats.latency_max = ts; - } + timestamp ts; + if (spl->cfg.use_stats) { + ts = platform_get_timestamp(); + } + test_key(&keybuf, + test_cfg->key_type, + insert_num, + thread_number, + test_cfg->semiseq_freq, + test_cfg->key_size, + test_cfg->period); + generate_test_message(test_cfg->gen, insert_num, &msg); + platform_status rc = core_insert(spl, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); + platform_assert_status_ok(rc); + if (spl->cfg.use_stats) { + ts = platform_timestamp_elapsed(ts); + if (ts > params->insert_stats.latency_max) { + params->insert_stats.latency_max = ts; } + } - // Throttle thread based on insert rate if needed. - if (insert_rate != 0) { - timestamp now = platform_get_timestamp(); - if (now <= next_check_time) { - num_inserts++; - if (num_inserts >= insert_rate) { - platform_sleep_ns(next_check_time - now); - } - } else { - // reset and check again after 10 msec. - num_inserts = 0; - next_check_time = now + (USEC_TO_NSEC(10000)); + // Throttle thread based on insert rate if needed. + if (insert_rate != 0) { + timestamp now = platform_get_timestamp(); + if (now <= next_check_time) { + num_inserts++; + if (num_inserts >= insert_rate) { + platform_sleep_ns(next_check_time - now); } + } else { + // reset and check again after 10 msec. + num_inserts = 0; + next_check_time = now + (USEC_TO_NSEC(10000)); } } } @@ -213,11 +174,7 @@ test_trunk_insert_thread(void *arg) out: merge_accumulator_deinit(&msg); params->rc = STATUS_OK; - platform_free(platform_get_heap_id(), insert_base); - for (uint64 i = 0; i < num_tables; i++) { - core_handle *spl = &spl_tables[i]; - core_perform_tasks(spl); - } + core_perform_tasks(spl); } /* @@ -228,118 +185,88 @@ test_trunk_lookup_thread(void *arg) { test_splinter_thread_params *params = (test_splinter_thread_params *)arg; - core_handle *spl_tables = params->spl; + core_handle *spl = params->spl; const test_config *test_cfg = params->test_cfg; - const uint64 *total_ops = params->total_ops; + const uint64 total_ops = params->total_ops; uint64 *curr_op = params->curr_op; uint64 op_granularity = params->op_granularity; uint64 thread_number = params->thread_number; bool32 expected_found = params->expected_found; - uint8 num_tables = params->num_tables; + test_async_lookup *async_lookup = params->async_lookup; verify_tuple_arg vtarg = {.expected_found = expected_found, .stats = ¶ms->lookup_stats[ASYNC_LU]}; platform_heap_id heap_id = platform_get_heap_id(); - platform_assert(num_tables <= 8); - uint64 *lookup_base = TYPED_ARRAY_ZALLOC(heap_id, lookup_base, num_tables); - uint8 done = 0; - lookup_result data; lookup_result_init(&data, NULL, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + uint64 lookup_base = 0; while (1) { - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - if (test_is_done(done, spl_idx)) { - continue; - } - test_print_progress(¶ms->progress, - lookup_base[spl_idx] / (total_ops[spl_idx] / 100), - PLATFORM_CR "Thread %lu lookups %3lu%% complete " - "for table %u", - thread_number, - lookup_base[spl_idx] / (total_ops[spl_idx] / 100), - spl_idx); - lookup_base[spl_idx] = - __sync_fetch_and_add(&curr_op[spl_idx], op_granularity); - if (lookup_base[spl_idx] >= total_ops[spl_idx]) { - test_set_done(&done, spl_idx); - } - if (test_all_done(done, num_tables)) { - goto out; - } + test_print_progress(¶ms->progress, + lookup_base / (total_ops / 100), + PLATFORM_CR "Thread %lu lookups %3lu%% complete", + thread_number, + lookup_base / (total_ops / 100)); + lookup_base = __sync_fetch_and_add(curr_op, op_granularity); + if (lookup_base >= total_ops) { + goto out; } DECLARE_AUTO_KEY_BUFFER(keybuf, heap_id); for (uint64 op_offset = 0; op_offset != op_granularity; op_offset++) { - uint8 spl_idx; - for (spl_idx = 0; spl_idx < num_tables; spl_idx++) { - if (test_is_done(done, spl_idx)) { - continue; - } - core_handle *spl = &spl_tables[spl_idx]; - test_async_lookup *async_lookup = params->async_lookup[spl_idx]; - test_async_ctxt *ctxt; - uint64 lookup_num = lookup_base[spl_idx] + op_offset; - timestamp ts; - - if (async_lookup->max_async_inflight == 0) { - platform_status rc; - - test_key(&keybuf, - test_cfg[spl_idx].key_type, - lookup_num, - thread_number, - test_cfg[spl_idx].semiseq_freq, - test_cfg[spl_idx].key_size, - test_cfg[spl_idx].period); - ts = platform_get_timestamp(); - lookup_result_set_data_config(&data, spl->cfg.data_cfg); - rc = core_lookup(spl, key_buffer_key(&keybuf), &data); - ts = platform_timestamp_elapsed(ts); - if (ts > params->lookup_stats[SYNC_LU].latency_max) { - params->lookup_stats[SYNC_LU].latency_max = ts; - } - platform_assert(SUCCESS(rc)); - verify_tuple(spl, - test_cfg->gen, - lookup_num, - key_buffer_key(&keybuf), - merge_accumulator_to_message( - lookup_result_accumulator(&data)), - expected_found); - } else { - ctxt = test_async_ctxt_get(spl, async_lookup, &vtarg); - test_key(&ctxt->key, - test_cfg[spl_idx].key_type, - lookup_num, - thread_number, - test_cfg[spl_idx].semiseq_freq, - test_cfg[spl_idx].key_size, - test_cfg[spl_idx].period); - ctxt->lookup_num = lookup_num; - async_ctxt_submit(spl, - async_lookup, - ctxt, - ¶ms->lookup_stats[ASYNC_LU].latency_max, - verify_tuple_callback, - &vtarg); + test_async_ctxt *ctxt; + uint64 lookup_num = lookup_base + op_offset; + timestamp ts; + + if (async_lookup->max_async_inflight == 0) { + platform_status rc; + + test_key(&keybuf, + test_cfg->key_type, + lookup_num, + thread_number, + test_cfg->semiseq_freq, + test_cfg->key_size, + test_cfg->period); + ts = platform_get_timestamp(); + lookup_result_set_data_config(&data, spl->cfg.data_cfg); + rc = core_lookup(spl, key_buffer_key(&keybuf), &data); + ts = platform_timestamp_elapsed(ts); + if (ts > params->lookup_stats[SYNC_LU].latency_max) { + params->lookup_stats[SYNC_LU].latency_max = ts; } + platform_assert(SUCCESS(rc)); + verify_tuple( + spl, + test_cfg->gen, + lookup_num, + key_buffer_key(&keybuf), + merge_accumulator_to_message(lookup_result_accumulator(&data)), + expected_found); + } else { + ctxt = test_async_ctxt_get(spl, async_lookup, &vtarg); + test_key(&ctxt->key, + test_cfg->key_type, + lookup_num, + thread_number, + test_cfg->semiseq_freq, + test_cfg->key_size, + test_cfg->period); + ctxt->lookup_num = lookup_num; + async_ctxt_submit(spl, + async_lookup, + ctxt, + ¶ms->lookup_stats[ASYNC_LU].latency_max, + verify_tuple_callback, + &vtarg); } } - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - if (test_is_done(done, spl_idx)) { - continue; - } - core_handle *spl = &spl_tables[spl_idx]; - test_async_lookup *async_lookup = params->async_lookup[spl_idx]; - test_wait_for_inflight(spl, async_lookup, &vtarg); - } + test_wait_for_inflight(spl, async_lookup, &vtarg); } out: lookup_result_deinit(&data); params->rc = STATUS_OK; - platform_free(platform_get_heap_id(), lookup_base); } static void @@ -360,21 +287,16 @@ test_trunk_range_thread(void *arg) { test_splinter_thread_params *params = (test_splinter_thread_params *)arg; - core_handle *spl_tables = params->spl; + core_handle *spl = params->spl; const test_config *test_cfg = params->test_cfg; - const uint64 *total_ops = params->total_ops; + const uint64 total_ops = params->total_ops; uint64 *curr_op = params->curr_op; uint64 op_granularity = params->op_granularity; uint64 thread_number = params->thread_number; uint64 min_range_length = params->min_range_length; uint64 max_range_length = params->max_range_length; - uint8 num_tables = params->num_tables; platform_heap_id heap_id = platform_get_heap_id(); - platform_assert(num_tables <= 8); - uint64 *range_base = TYPED_ARRAY_ZALLOC(heap_id, range_base, num_tables); - uint8 done = 0; - bool32 verbose_progress = test_show_verbose_progress(test_cfg->test_exec_cfg); uint64 test_start_time = platform_get_timestamp(); @@ -383,83 +305,65 @@ test_trunk_range_thread(void *arg) DECLARE_AUTO_KEY_BUFFER(start_key, heap_id); + uint64 range_base = 0; while (1) { - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - if (test_is_done(done, spl_idx)) { - continue; - } - int pct_done = (range_base[spl_idx] / (total_ops[spl_idx] / 100)); - char *newmsg = ""; - - if (verbose_progress) { - // For all threads other than thread 0, periodically emit a message - if (thread_number && pct_done && ((pct_done % 10) == 0)) { - uint64 delta_time = platform_timestamp_elapsed(start_time); - uint64 total_time = platform_timestamp_elapsed(test_start_time); - snprintf(progress_msg, - sizeof(progress_msg), - " ... delta=%lus, total time=%lus\n", - NSEC_TO_SEC(delta_time), - NSEC_TO_SEC(total_time)); - newmsg = progress_msg; - - // Reset ... for next batch of progress messages - start_time = platform_get_timestamp(); - } - } - test_print_progress(¶ms->progress, - range_base[spl_idx] / (total_ops[spl_idx] / 100), - PLATFORM_CR - "Thread %lu range lookups %3lu%% " - "complete for table %u, range_base=%lu%s", - thread_number, - range_base[spl_idx] / (total_ops[spl_idx] / 100), - spl_idx, - range_base[spl_idx], - newmsg); - - range_base[spl_idx] = - __sync_fetch_and_add(&curr_op[spl_idx], op_granularity); - if (range_base[spl_idx] >= total_ops[spl_idx]) { - test_set_done(&done, spl_idx); - } - if (test_all_done(done, num_tables)) { - goto out; + uint64 pct_done = (range_base / (total_ops / 100)); + char *newmsg = ""; + + if (verbose_progress) { + // For all threads other than thread 0, periodically emit a message + if (thread_number && pct_done && ((pct_done % 10) == 0)) { + uint64 delta_time = platform_timestamp_elapsed(start_time); + uint64 total_time = platform_timestamp_elapsed(test_start_time); + snprintf(progress_msg, + sizeof(progress_msg), + " ... delta=%lus, total time=%lus\n", + NSEC_TO_SEC(delta_time), + NSEC_TO_SEC(total_time)); + newmsg = progress_msg; + + // Reset ... for next batch of progress messages + start_time = platform_get_timestamp(); } } + test_print_progress(¶ms->progress, + pct_done, + PLATFORM_CR "Thread %lu range lookups %3lu%% " + "complete, range_base=%lu%s", + thread_number, + pct_done, + range_base, + newmsg); + + range_base = __sync_fetch_and_add(curr_op, op_granularity); + if (range_base >= total_ops) { + goto out; + } for (uint64 op_offset = 0; op_offset != op_granularity; op_offset++) { - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - if (test_is_done(done, spl_idx)) { - continue; - } - core_handle *spl = &spl_tables[spl_idx]; + uint64 range_num = range_base + op_offset; + test_key(&start_key, + test_cfg->key_type, + range_num, + thread_number, + test_cfg->semiseq_freq, + test_cfg->key_size, + test_cfg->period); + uint64 range_tuples = + test_range(range_num, min_range_length, max_range_length); + platform_status rc = + core_apply_to_range(spl, + key_buffer_key(&start_key), + range_tuples, + count_range_tuple_func, + ¶ms->range_tuples_returned); + platform_assert_status_ok(rc); - uint64 range_num = range_base[spl_idx] + op_offset; - test_key(&start_key, - test_cfg[spl_idx].key_type, - range_num, - thread_number, - test_cfg[spl_idx].semiseq_freq, - test_cfg[spl_idx].key_size, - test_cfg[spl_idx].period); - uint64 range_tuples = - test_range(range_num, min_range_length, max_range_length); - platform_status rc = - core_apply_to_range(spl, - key_buffer_key(&start_key), - range_tuples, - count_range_tuple_func, - ¶ms->range_tuples_returned); - platform_assert_status_ok(rc); - - params->range_lookups_done++; - } + params->range_lookups_done++; } } out: params->rc = STATUS_OK; - platform_free(platform_get_heap_id(), range_base); } /* @@ -477,78 +381,59 @@ static bool32 advance_base(test_splinter_thread_params *params, uint64 *curr_op, uint64 *base, - uint8 *done, random_state *rs, test_splinter_pthread_op_type type) { - const uint64 *total_ops = params->total_ops; - const uint64 op_granularity = params->op_granularity; - const uint8 num_tables = params->num_tables; - const uint8 lookup_positive_pct = params->lookup_positive_pct; - - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - if (test_is_done(*done, spl_idx)) { - continue; + const uint64 total_ops = params->total_ops; + const uint64 op_granularity = params->op_granularity; + const uint8 lookup_positive_pct = params->lookup_positive_pct; + + if (type == OP_INSERT) { + test_print_progress(¶ms->progress, + *base / (total_ops / 100), + PLATFORM_CR "inserting/lookups %3lu%% complete", + *base / (total_ops / 100)); + *base = __sync_fetch_and_add(curr_op, op_granularity); + if (*base >= total_ops) { + return TRUE; } - - if (type == OP_INSERT) { - test_print_progress(¶ms->progress, - base[spl_idx] / (total_ops[spl_idx] / 100), - PLATFORM_CR "inserting/lookups %3lu%% complete " - "for table %u", - base[spl_idx] / (total_ops[spl_idx] / 100), - spl_idx); - base[spl_idx] = - __sync_fetch_and_add(&curr_op[spl_idx], op_granularity); - if (base[spl_idx] >= total_ops[spl_idx]) { - test_set_done(done, spl_idx); - } - if (test_all_done(*done, num_tables)) { - return TRUE; + } else { + /* lookup */ + uint64 random_base; + uint64 local_curr_op = *curr_op; + // pick an random number to determine + // if positive or negative lookup + // [0, lookup_positive_pct) is positive lookup + if (random_next_uint64(rs) % 100 < lookup_positive_pct) { + if (local_curr_op == op_granularity) { + random_base = 0; + } else { + // positive lookup by selecting random base + // [op_granularity, local_curr_op) + random_base = + ((random_next_uint64(rs) % (local_curr_op / op_granularity - 1)) + + 1) + * op_granularity; } } else { - /* lookup */ - uint64 random_base; - uint64 local_curr_op = curr_op[spl_idx]; - // pick an random number to determine - // if positive or negative lookup - // [0, lookup_positive_pct) is positive lookup - if (random_next_uint64(rs) % 100 < lookup_positive_pct) { - if (local_curr_op == op_granularity) { - random_base = 0; - } else { - // positive lookup by selecting random base - // [op_granularity, local_curr_op) - random_base = ((random_next_uint64(rs) - % (local_curr_op / op_granularity - 1)) - + 1) - * op_granularity; - } + if (local_curr_op >= total_ops - op_granularity) { + // one interval left, + // may lookup the key never be inserted + random_base = + (random_next_uint64(rs) / op_granularity) * op_granularity + + local_curr_op; } else { - if (local_curr_op >= total_ops[spl_idx] - op_granularity) { - // one interval left, - // may lookup the key never be inserted - random_base = - (random_next_uint64(rs) / op_granularity) * op_granularity - + local_curr_op; - } else { - // negative lookup by select random base - // [local_curr_op + op_granularity, total_ops - op_granularity] - random_base = - ((random_next_uint64(rs) - % ((total_ops[spl_idx] - local_curr_op) / op_granularity - - 1)) - + 1) - * op_granularity - + local_curr_op; - } + // negative lookup by select random base + // [local_curr_op + op_granularity, total_ops - op_granularity] + random_base = + ((random_next_uint64(rs) + % ((total_ops - local_curr_op) / op_granularity - 1)) + + 1) + * op_granularity + + local_curr_op; } - base[spl_idx] = random_base; } - } - - if (test_all_done(*done, num_tables)) { - return TRUE; + *base = random_base; } return FALSE; @@ -564,17 +449,15 @@ advance_base(test_splinter_thread_params *params, */ static void do_operation(test_splinter_thread_params *params, - const uint64 *base, + uint64 base, uint64 num_ops, uint64 op_offset, - const uint8 *done, bool32 is_insert) { - core_handle *spl_tables = params->spl; + core_handle *spl = params->spl; const test_config *test_cfg = params->test_cfg; uint64 op_granularity = params->op_granularity; uint64 thread_number = params->thread_number; - uint8 num_tables = params->num_tables; verify_tuple_arg vtarg = {.stats_only = TRUE, .stats = ¶ms->lookup_stats[ASYNC_LU]}; platform_heap_id heap_id = platform_get_heap_id(); @@ -587,82 +470,76 @@ do_operation(test_splinter_thread_params *params, for (uint64 op_idx = op_offset; op_idx != op_offset + num_ops; op_idx++) { if (op_idx >= op_granularity) { - return; + break; } - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - if (test_is_done(*done, spl_idx)) { - continue; + uint64 op_num = base + op_idx; + timestamp ts; + + if (is_insert) { + test_key(&keybuf, + test_cfg->key_type, + op_num, + thread_number, + test_cfg->semiseq_freq, + test_cfg->key_size, + test_cfg->period); + generate_test_message(test_cfg->gen, op_num, &msg); + ts = platform_get_timestamp(); + platform_status rc = core_insert(spl, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); + platform_assert_status_ok(rc); + ts = platform_timestamp_elapsed(ts); + params->insert_stats.duration += ts; + if (ts > params->insert_stats.latency_max) { + params->insert_stats.latency_max = ts; } - core_handle *spl = &spl_tables[spl_idx]; - uint64 op_num = base[spl_idx] + op_idx; - timestamp ts; + } else { + test_async_lookup *async_lookup = params->async_lookup; + test_async_ctxt *ctxt; + + if (async_lookup->max_async_inflight == 0) { + platform_status rc; - if (is_insert) { test_key(&keybuf, - test_cfg[spl_idx].key_type, + test_cfg->key_type, op_num, thread_number, - test_cfg[spl_idx].semiseq_freq, - test_cfg[spl_idx].key_size, - test_cfg[spl_idx].period); - generate_test_message(test_cfg->gen, op_num, &msg); - ts = platform_get_timestamp(); - platform_status rc = core_insert(spl, - key_buffer_key(&keybuf), - merge_accumulator_to_message(&msg), - NULL); - platform_assert_status_ok(rc); + test_cfg->semiseq_freq, + test_cfg->key_size, + test_cfg->period); + ts = platform_get_timestamp(); + lookup_result_set_data_config(&lookup, spl->cfg.data_cfg); + rc = core_lookup(spl, key_buffer_key(&keybuf), &lookup); + platform_assert(SUCCESS(rc)); ts = platform_timestamp_elapsed(ts); - params->insert_stats.duration += ts; - if (ts > params->insert_stats.latency_max) { - params->insert_stats.latency_max = ts; + if (ts > params->lookup_stats[SYNC_LU].latency_max) { + params->lookup_stats[SYNC_LU].latency_max = ts; } - } else { - test_async_lookup *async_lookup = params->async_lookup[spl_idx]; - test_async_ctxt *ctxt; - - if (async_lookup->max_async_inflight == 0) { - platform_status rc; - - test_key(&keybuf, - test_cfg[spl_idx].key_type, - op_num, - thread_number, - test_cfg[spl_idx].semiseq_freq, - test_cfg[spl_idx].key_size, - test_cfg[spl_idx].period); - ts = platform_get_timestamp(); - lookup_result_set_data_config(&lookup, spl->cfg.data_cfg); - rc = core_lookup(spl, key_buffer_key(&keybuf), &lookup); - platform_assert(SUCCESS(rc)); - ts = platform_timestamp_elapsed(ts); - if (ts > params->lookup_stats[SYNC_LU].latency_max) { - params->lookup_stats[SYNC_LU].latency_max = ts; - } - bool32 found = lookup_result_found(&lookup); - if (found) { - params->lookup_stats[SYNC_LU].num_found++; - } else { - params->lookup_stats[SYNC_LU].num_not_found++; - } + bool32 found = lookup_result_found(&lookup); + if (found) { + params->lookup_stats[SYNC_LU].num_found++; } else { - ctxt = test_async_ctxt_get(spl, async_lookup, &vtarg); - test_key(&ctxt->key, - test_cfg[spl_idx].key_type, - op_num, - thread_number, - test_cfg[spl_idx].semiseq_freq, - test_cfg[spl_idx].key_size, - test_cfg[spl_idx].period); - ctxt->lookup_num = op_num; - async_ctxt_submit(spl, - async_lookup, - ctxt, - ¶ms->lookup_stats[ASYNC_LU].latency_max, - verify_tuple_callback, - &vtarg); + params->lookup_stats[SYNC_LU].num_not_found++; } + } else { + ctxt = test_async_ctxt_get(spl, async_lookup, &vtarg); + test_key(&ctxt->key, + test_cfg->key_type, + op_num, + thread_number, + test_cfg->semiseq_freq, + test_cfg->key_size, + test_cfg->period); + ctxt->lookup_num = op_num; + async_ctxt_submit(spl, + async_lookup, + ctxt, + ¶ms->lookup_stats[ASYNC_LU].latency_max, + verify_tuple_callback, + &vtarg); } } } @@ -686,31 +563,27 @@ test_trunk_insert_lookup_thread(void *arg) { test_splinter_thread_params *params = (test_splinter_thread_params *)arg; - core_handle *spl_tables = params->spl; - uint8 num_tables = params->num_tables; - uint64 op_granularity = params->op_granularity; - uint64 seed = params->seed; + core_handle *spl = params->spl; + uint64 op_granularity = params->op_granularity; + uint64 seed = params->seed; + verify_tuple_arg vtarg = {.stats_only = TRUE, + .stats = ¶ms->lookup_stats[ASYNC_LU]}; - platform_assert(num_tables <= 8); - - uint64 *bases[NUM_OP_TYPES]; + uint64 bases[NUM_OP_TYPES]; uint64 granularities[NUM_OP_TYPES]; uint64 offsets[NUM_OP_TYPES]; - uint8 insert_done = 0; - uint64 num_ops = 0; + uint64 num_ops = 0; random_state rs; random_init(&rs, seed, 0); for (uint8 i = 0; i < NUM_OP_TYPES; i++) { - bases[i] = - TYPED_ARRAY_ZALLOC(platform_get_heap_id(), bases[i], num_tables); - + bases[i] = 0; granularities[i] = params->num_ops_per_thread[i]; offsets[i] = 0; } for (uint8 i = 0; i < NUM_OP_TYPES; i++) { - advance_base(params, params->curr_op, bases[i], &insert_done, &rs, i); + advance_base(params, params->curr_op, &bases[i], &rs, i); } while (1) { @@ -730,7 +603,6 @@ test_trunk_insert_lookup_thread(void *arg) bases[op_type], num_ops, offsets[op_type], - &insert_done, op_type == OP_INSERT); offsets[op_type] += num_ops; } @@ -739,12 +611,8 @@ test_trunk_insert_lookup_thread(void *arg) || offsets[op_type] >= op_granularity) { num_ops = granularities[op_type] - num_ops; - if (advance_base(params, - params->curr_op, - bases[op_type], - &insert_done, - &rs, - op_type)) + if (advance_base( + params, params->curr_op, &bases[op_type], &rs, op_type)) { goto out; } @@ -755,7 +623,6 @@ test_trunk_insert_lookup_thread(void *arg) bases[op_type], num_ops, offsets[op_type], - &insert_done, op_type == OP_INSERT); offsets[op_type] += num_ops; } @@ -764,100 +631,76 @@ test_trunk_insert_lookup_thread(void *arg) } out: - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - verify_tuple_arg vtarg = {.stats_only = TRUE, - .stats = ¶ms->lookup_stats[ASYNC_LU]}; - test_async_lookup *async_lookup = params->async_lookup[spl_idx]; - test_wait_for_inflight(spl, async_lookup, &vtarg); - } + test_wait_for_inflight(spl, params->async_lookup, &vtarg); params->rc = STATUS_OK; - for (uint8 i = 0; i < NUM_OP_TYPES; i++) { - platform_free(platform_get_heap_id(), bases[i]); - } } +/* + * The unified superblock is per-device: core owns one superblock per device, so + * these tests drive exactly one table. + */ static platform_status -test_trunk_create_tables(core_handle **spl_handles, - system_config *cfg, - allocator *al, - cache *cc[], - task_system *ts, - platform_heap_id hid, - uint8 num_tables, - uint8 num_caches) +test_trunk_create_table(core_handle **spl_handle, + system_config *cfg, + allocator *al, + cache *cc, + task_system *ts, + platform_heap_id hid) { - core_handle *spl_tables = TYPED_ARRAY_ZALLOC(hid, spl_tables, num_tables); - if (spl_tables == NULL) { + // These functional tests use rc_allocator exclusively; core_mkfs needs the + // io handle, which the allocator holds. + io_handle *io = ((rc_allocator *)al)->io; + + core_handle *spl = TYPED_ZALLOC(hid, spl); + if (spl == NULL) { return STATUS_NO_MEMORY; } - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - cache *cache_to_use = num_caches > 1 ? cc[spl_idx] : *cc; - platform_status rc = core_mkfs(&spl_tables[spl_idx], - &cfg[spl_idx].splinter_cfg, - al, - cache_to_use, - ts, - test_generate_allocator_root_id(), - hid); - if (!SUCCESS(rc)) { - for (uint8 del_idx = 0; del_idx < spl_idx; del_idx++) { - core_destroy(&spl_tables[del_idx]); - } - platform_free(hid, spl_tables); - return rc; - } + platform_status rc = core_mkfs(spl, + &cfg->splinter_cfg, + al, + cc, + io, + ts, + test_generate_allocator_root_id(), + hid); + if (!SUCCESS(rc)) { + platform_free(hid, spl); + return rc; } - *spl_handles = spl_tables; + *spl_handle = spl; return STATUS_OK; } static void -test_trunk_destroy_tables(core_handle *spl_tables, - platform_heap_id hid, - uint8 num_tables) +test_trunk_destroy_table(core_handle *spl, platform_heap_id hid) { - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_destroy(&spl_tables[spl_idx]); - } - platform_free(hid, spl_tables); + core_destroy(spl); + platform_free(hid, spl); } /* - * compute_per_table_inserts() -- + * compute_num_inserts() -- * - * Helper function to compute the # of inserts for each table, based on the + * Helper function to compute the # of inserts for the table, based on the * configuration specified. - * - * Returns: Total # of inserts to-be-done in the workload */ static uint64 -compute_per_table_inserts(uint64 *per_table_inserts, // OUT - system_config *cfg, // IN - test_config *test_cfg, // IN - uint8 num_tables) +compute_num_inserts(test_config *test_cfg) // IN { - uint64 tuple_size; - uint64 num_inserts; uint64 num_inserts_cfg = test_cfg->test_exec_cfg->num_inserts; - uint64 total_inserts = 0; - - for (uint8 i = 0; i < num_tables; i++) { - tuple_size = - test_cfg[i].key_size + generator_average_message_size(test_cfg->gen); - num_inserts = (num_inserts_cfg ? num_inserts_cfg - : test_cfg[i].tree_size / tuple_size); - if (test_cfg[i].key_type == TEST_PERIODIC) { - test_cfg[i].period = num_inserts; - num_inserts *= test_cfg[i].num_periods; - } - per_table_inserts[i] = ROUNDUP(num_inserts, TEST_INSERT_GRANULARITY); - total_inserts += per_table_inserts[i]; + uint64 tuple_size = + test_cfg->key_size + generator_average_message_size(test_cfg->gen); + uint64 num_inserts = + (num_inserts_cfg ? num_inserts_cfg : test_cfg->tree_size / tuple_size); + + if (test_cfg->key_type == TEST_PERIODIC) { + test_cfg->period = num_inserts; + num_inserts *= test_cfg->num_periods; } - return total_inserts; + return ROUNDUP(num_inserts, TEST_INSERT_GRANULARITY); } /* @@ -867,11 +710,10 @@ compute_per_table_inserts(uint64 *per_table_inserts, // OUT */ static void load_thread_params(test_splinter_thread_params *params, - core_handle *spl_tables, + core_handle *spl, test_config *test_cfg, - uint64 *per_table_inserts, + uint64 num_inserts, uint64 *curr_op, - uint8 num_tables, task_system *ts, uint64 insert_rate, uint64 num_insert_threads, @@ -879,13 +721,12 @@ load_thread_params(test_splinter_thread_params *params, bool32 is_parallel) { for (uint64 i = 0; i < num_threads; i++) { - params[i].spl = spl_tables; + params[i].spl = spl; params[i].test_cfg = test_cfg; - params[i].total_ops = per_table_inserts; + params[i].total_ops = num_inserts; params[i].curr_op = curr_op; params[i].op_granularity = TEST_INSERT_GRANULARITY; params[i].thread_number = i; - params[i].num_tables = num_tables; params[i].ts = ts; if (!is_parallel) { @@ -923,42 +764,34 @@ do_n_thread_creates(const char *thread_type, /* * do_n_async_ctxt_inits() -- * - * Helper function to setup Async contexts for num_threads threads, each thread - * processing num_tables tables. This async-lookup context is tied to each - * table. Lookup execution for each table needs to access this context before it - * can check if (max_async_inflight == 0). + * Helper function to setup Async contexts for num_threads threads. Lookup + * execution needs to access this context before it can check if + * (max_async_inflight == 0). */ static void do_n_async_ctxt_inits(platform_heap_id hid, uint64 num_threads, - uint8 num_tables, uint64 max_async_inflight, - system_config *cfg, test_splinter_thread_params *params) { for (uint64 i = 0; i < num_threads; i++) { - for (uint8 j = 0; j < num_tables; j++) { - async_ctxt_init(hid, max_async_inflight, ¶ms[i].async_lookup[j]); - } + async_ctxt_init(hid, max_async_inflight, ¶ms[i].async_lookup); } } /* * do_n_async_ctxt_deinits() -- * - * Helper function to dismantle n-Async contexts for m-tables. + * Helper function to dismantle n-Async contexts. */ static void do_n_async_ctxt_deinits(platform_heap_id hid, uint64 num_threads, - uint8 num_tables, test_splinter_thread_params *params) { for (uint64 i = 0; i < num_threads; i++) { - for (uint8 j = 0; j < num_tables; j++) { - async_ctxt_deinit(hid, params[i].async_lookup[j]); - params[i].async_lookup[j] = NULL; - } + async_ctxt_deinit(hid, params[i].async_lookup); + params[i].async_lookup = NULL; } } @@ -971,23 +804,19 @@ do_n_async_ctxt_deinits(platform_heap_id hid, */ static platform_status splinter_perf_inserts(platform_heap_id hid, - system_config *cfg, test_config *test_cfg, - core_handle *spl_tables, - cache *cc[], + core_handle *spl, + cache *cc, task_system *ts, test_splinter_thread_params *params, - uint64 *per_table_inserts, uint64 *curr_op, uint64 num_insert_threads, uint64 num_threads, - uint8 num_tables, uint64 insert_rate, uint64 *total_inserts) { platform_status rc; - *total_inserts = - compute_per_table_inserts(per_table_inserts, cfg, test_cfg, num_tables); + *total_inserts = compute_num_inserts(test_cfg); platform_default_log("%s() starting num_insert_threads=%lu, num_threads=%lu" ", num_inserts=%lu (~%lu million) ...\n", @@ -998,11 +827,10 @@ splinter_perf_inserts(platform_heap_id hid, (*total_inserts / MILLION)); load_thread_params(params, - spl_tables, + spl, test_cfg, - per_table_inserts, + *total_inserts, curr_op, - num_tables, ts, insert_rate, num_insert_threads, @@ -1038,7 +866,7 @@ splinter_perf_inserts(platform_heap_id hid, uint64 total_time = platform_timestamp_elapsed(start_time); timestamp insert_latency_max = 0; uint64 read_io_bytes, write_io_bytes; - cache_io_stats(cc[0], &read_io_bytes, &write_io_bytes); + cache_io_stats(cc, &read_io_bytes, &write_io_bytes); uint64 io_mib = (read_io_bytes + write_io_bytes) / MiB; uint64 bandwidth = (NSEC_TO_SEC(total_time) ? (io_mib / NSEC_TO_SEC(total_time)) : 0); @@ -1069,15 +897,12 @@ splinter_perf_inserts(platform_heap_id hid, NSEC_TO_MSEC(insert_latency_max)); } - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - cache_assert_free(spl->cc); - core_print_insertion_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - core_print_space_use(Platform_default_log_handle, spl); - cache_reset_stats(spl->cc); - // trunk_print(Platform_default_log_handle, spl); - } + cache_assert_free(spl->cc); + core_print_insertion_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + core_print_space_use(Platform_default_log_handle, spl); + cache_reset_stats(spl->cc); + platform_default_log("%s() completed.\n\n", __FUNCTION__); return rc; } @@ -1091,13 +916,11 @@ splinter_perf_inserts(platform_heap_id hid, */ static platform_status splinter_perf_lookups(platform_heap_id hid, - system_config *cfg, test_config *test_cfg, - core_handle *spl_tables, + core_handle *spl, task_system *ts, test_splinter_thread_params *params, uint64 num_lookup_threads, - uint8 num_tables, uint32 max_async_inflight, uint64 total_inserts) { @@ -1109,8 +932,7 @@ splinter_perf_lookups(platform_heap_id hid, uint64 start_time = platform_get_timestamp(); platform_status rc; - do_n_async_ctxt_inits( - hid, num_lookup_threads, num_tables, max_async_inflight, cfg, params); + do_n_async_ctxt_inits(hid, num_lookup_threads, max_async_inflight, params); rc = do_n_thread_creates("lookup_thread", num_lookup_threads, @@ -1140,7 +962,7 @@ splinter_perf_lookups(platform_heap_id hid, timestamp sync_lookup_latency_max = 0; timestamp async_lookup_latency_max = 0; - do_n_async_ctxt_deinits(hid, num_lookup_threads, num_tables, params); + do_n_async_ctxt_deinits(hid, num_lookup_threads, params); for (uint64 i = 0; i < num_lookup_threads; i++) { num_async_lookups += params[i].lookup_stats[ASYNC_LU].num_found @@ -1172,14 +994,12 @@ splinter_perf_lookups(platform_heap_id hid, platform_default_log("max lookup latency ns (sync=%lu, async=%lu)\n", sync_lookup_latency_max, async_lookup_latency_max); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - cache_assert_free(spl->cc); - core_print_insertion_stats(Platform_default_log_handle, spl); - core_print_lookup_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - cache_reset_stats(spl->cc); - } + cache_assert_free(spl->cc); + core_print_insertion_stats(Platform_default_log_handle, spl); + core_print_lookup_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + cache_reset_stats(spl->cc); + platform_default_log("%s() completed.\n\n", __FUNCTION__); return rc; } @@ -1195,13 +1015,11 @@ splinter_perf_lookups(platform_heap_id hid, static platform_status splinter_perf_range_lookups(platform_heap_id hid, test_config *test_cfg, - core_handle *spl_tables, + core_handle *spl, task_system *ts, test_splinter_thread_params *params, - uint64 *per_table_inserts, - uint64 *per_table_ranges, + uint64 num_inserts, uint64 num_range_threads, - uint8 num_tables, trunk_range_perf_params *range_perf) { const char *range_descr = range_perf->range_perf_descr; @@ -1222,24 +1040,15 @@ splinter_perf_range_lookups(platform_heap_id hid, bool32 verbose_progress = test_show_verbose_progress(test_cfg->test_exec_cfg); - uint64 total_ranges = 0; - for (uint8 i = 0; i < num_tables; i++) { - per_table_ranges[i] = - ROUNDUP(per_table_inserts[i] / num_ranges, TEST_RANGE_GRANULARITY); - total_ranges += per_table_ranges[i]; - - if (verbose_progress) { - platform_default_log(" Table[%d], per_table_inserts=%lu" - ", per_table_ranges=%lu, total_ranges=%lu\n", - i, - per_table_inserts[i], - per_table_ranges[i], - total_ranges); - } + uint64 num_ranges_total = + ROUNDUP(num_inserts / num_ranges, TEST_RANGE_GRANULARITY); + if (verbose_progress) { + platform_default_log( + " num_inserts=%lu, num_ranges=%lu\n", num_inserts, num_ranges_total); } for (uint64 i = 0; i < num_range_threads; i++) { - params[i].total_ops = per_table_ranges; + params[i].total_ops = num_ranges_total; params[i].op_granularity = TEST_RANGE_GRANULARITY; params[i].min_range_length = min_range_length; params[i].max_range_length = max_range_length; @@ -1248,10 +1057,10 @@ splinter_perf_range_lookups(platform_heap_id hid, if (verbose_progress) { platform_default_log(" Range thread[%lu] " - "total_ops for table0=%lu, op_granularity=%lu" + "total_ops=%lu, op_granularity=%lu" ", min_range_length=%lu, max_range_length=%lu\n", i, - *params[i].total_ops, + params[i].total_ops, params[i].op_granularity, params[i].min_range_length, params[i].max_range_length); @@ -1325,13 +1134,11 @@ splinter_perf_range_lookups(platform_heap_id hid, (total_time ? SEC_TO_NSEC(num_range_lookups) / total_time : 0), (total_time ? SEC_TO_NSEC(total_returned_tuples) / total_time : 0)); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - cache_assert_free(spl->cc); - core_print_lookup_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - cache_reset_stats(spl->cc); - } + cache_assert_free(spl->cc); + core_print_lookup_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + cache_reset_stats(spl->cc); + platform_default_log("%s() completed.\n\n", __FUNCTION__); return rc; } @@ -1355,36 +1162,27 @@ static platform_status test_splinter_perf(system_config *cfg, test_config *test_cfg, allocator *al, - cache *cc[], + cache *cc, uint64 num_insert_threads, uint64 num_lookup_threads, uint64 num_range_threads, uint32 max_async_inflight, task_system *ts, platform_heap_id hid, - uint8 num_tables, - uint8 num_caches, uint64 insert_rate) { - platform_default_log("splinter_test: SplinterDB performance test started " - "with %d tables\n", - num_tables); - core_handle *spl_tables; + platform_default_log("splinter_test: SplinterDB performance test started\n"); + core_handle *spl; platform_status rc; - rc = test_trunk_create_tables( - &spl_tables, cfg, al, cc, ts, hid, num_tables, num_caches); + rc = test_trunk_create_table(&spl, cfg, al, cc, ts, hid); if (!SUCCESS(rc)) { - platform_error_log("Failed to create splinter table(s): %s\n", + platform_error_log("Failed to create splinter table: %s\n", platform_status_to_string(rc)); return rc; } - uint64 *per_table_inserts = - TYPED_ARRAY_MALLOC(hid, per_table_inserts, num_tables); - uint64 *per_table_ranges = - TYPED_ARRAY_MALLOC(hid, per_table_ranges, num_tables); - uint64 *curr_op = TYPED_ARRAY_ZALLOC(hid, curr_op, num_tables); + uint64 curr_op = 0; uint64 num_threads = MAX(num_insert_threads, num_lookup_threads); num_threads = MAX(num_threads, num_range_threads); @@ -1398,17 +1196,14 @@ test_splinter_perf(system_config *cfg, // if nothing is being inserted. platform_assert(num_insert_threads > 0); rc = splinter_perf_inserts(hid, - cfg, test_cfg, - spl_tables, + spl, cc, ts, params, - per_table_inserts, - curr_op, + &curr_op, num_insert_threads, num_threads, - num_tables, insert_rate, &total_inserts); if (!SUCCESS(rc)) { @@ -1416,15 +1211,13 @@ test_splinter_perf(system_config *cfg, } if (num_lookup_threads > 0) { - ZERO_CONTENTS_N(curr_op, num_tables); - rc = splinter_perf_lookups(hid, - cfg, + curr_op = 0; + rc = splinter_perf_lookups(hid, test_cfg, - spl_tables, + spl, ts, params, num_lookup_threads, - num_tables, max_async_inflight, total_inserts); if (!SUCCESS(rc)) { @@ -1445,16 +1238,14 @@ test_splinter_perf(system_config *cfg, // clang-format on for (int rctr = 0; rctr < ARRAY_SIZE(perf_ranges); rctr++) { - ZERO_CONTENTS_N(curr_op, num_tables); - rc = splinter_perf_range_lookups(hid, + curr_op = 0; + rc = splinter_perf_range_lookups(hid, test_cfg, - spl_tables, + spl, ts, params, - per_table_inserts, - per_table_ranges, + total_inserts, num_range_threads, - num_tables, &perf_ranges[rctr]); if (!SUCCESS(rc)) { goto destroy_splinter; @@ -1463,15 +1254,10 @@ test_splinter_perf(system_config *cfg, } destroy_splinter: - test_trunk_destroy_tables(spl_tables, hid, num_tables); + test_trunk_destroy_table(spl, hid); platform_default_log("After destroy:\n"); - for (uint8 idx = 0; idx < num_caches; idx++) { - cache_print_stats(Platform_default_log_handle, cc[idx]); - } + cache_print_stats(Platform_default_log_handle, cc); platform_free(hid, params); - platform_free(hid, curr_op); - platform_free(hid, per_table_ranges); - platform_free(hid, per_table_inserts); return rc; } @@ -1479,51 +1265,36 @@ platform_status test_splinter_periodic(system_config *cfg, test_config *test_cfg, allocator *al, - cache *cc[], + cache *cc, uint64 num_insert_threads, uint64 num_lookup_threads, uint64 num_range_threads, uint32 max_async_inflight, task_system *ts, platform_heap_id hid, - uint8 num_tables, - uint8 num_caches, uint64 insert_rate) { platform_default_log( - "splinter_test: SplinterDB performance test (periodic) started with " - "%d tables\n", - num_tables); - core_handle *spl_tables; + "splinter_test: SplinterDB performance test (periodic) started\n"); + core_handle *spl; platform_status rc; - rc = test_trunk_create_tables( - &spl_tables, cfg, al, cc, ts, hid, num_tables, num_caches); + rc = test_trunk_create_table(&spl, cfg, al, cc, ts, hid); if (!SUCCESS(rc)) { - platform_error_log("Failed to create splinter table(s): %s\n", + platform_error_log("Failed to create splinter table: %s\n", platform_status_to_string(rc)); return rc; } - uint64 tuple_size, num_inserts; - uint64 *per_table_inserts = - TYPED_ARRAY_MALLOC(hid, per_table_inserts, num_tables); - uint64 *per_table_ranges = - TYPED_ARRAY_MALLOC(hid, per_table_ranges, num_tables); - uint64 *curr_op = TYPED_ARRAY_ZALLOC(hid, curr_op, num_tables); - uint64 total_inserts = 0; - - for (uint8 i = 0; i < num_tables; i++) { - tuple_size = - test_cfg[i].key_size + generator_average_message_size(test_cfg->gen); - num_inserts = test_cfg[i].tree_size / tuple_size; - if (test_cfg[i].key_type == TEST_PERIODIC) { - test_cfg[i].period = num_inserts; - num_inserts *= test_cfg[i].num_periods; - } - per_table_inserts[i] = ROUNDUP(num_inserts, TEST_INSERT_GRANULARITY); - total_inserts += per_table_inserts[i]; + uint64 curr_op = 0; + uint64 tuple_size = + test_cfg->key_size + generator_average_message_size(test_cfg->gen); + uint64 num_inserts = test_cfg->tree_size / tuple_size; + if (test_cfg->key_type == TEST_PERIODIC) { + test_cfg->period = num_inserts; + num_inserts *= test_cfg->num_periods; } + uint64 total_inserts = ROUNDUP(num_inserts, TEST_INSERT_GRANULARITY); uint64 num_threads = num_insert_threads; @@ -1537,17 +1308,15 @@ test_splinter_periodic(system_config *cfg, test_splinter_thread_params *params = TYPED_ARRAY_ZALLOC(hid, params, num_threads); for (uint64 i = 0; i < num_threads; i++) { - params[i].spl = spl_tables; + params[i].spl = spl; params[i].test_cfg = test_cfg; - params[i].total_ops = per_table_inserts; - params[i].curr_op = curr_op; + params[i].total_ops = total_inserts; + params[i].curr_op = &curr_op; params[i].op_granularity = TEST_INSERT_GRANULARITY; params[i].thread_number = i; params[i].expected_found = TRUE; - params[i].num_tables = num_tables; params[i].ts = ts; params[i].insert_rate = insert_rate / num_insert_threads; - params[i].expected_found = TRUE; } uint64 start_time = platform_get_timestamp(); @@ -1568,7 +1337,7 @@ test_splinter_periodic(system_config *cfg, uint64 total_time = platform_timestamp_elapsed(start_time); timestamp insert_latency_max = 0; uint64 read_io_bytes, write_io_bytes; - cache_io_stats(cc[0], &read_io_bytes, &write_io_bytes); + cache_io_stats(cc, &read_io_bytes, &write_io_bytes); uint64 io_mib = (read_io_bytes + write_io_bytes) / MiB; uint64 bandwidth = io_mib / NSEC_TO_SEC(total_time); @@ -1597,16 +1366,13 @@ test_splinter_periodic(system_config *cfg, NSEC_TO_MSEC(insert_latency_max)); } - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - cache_assert_free(spl->cc); - core_print_insertion_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - core_print_space_use(Platform_default_log_handle, spl); - cache_reset_stats(spl->cc); - } + cache_assert_free(spl->cc); + core_print_insertion_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + core_print_space_use(Platform_default_log_handle, spl); + cache_reset_stats(spl->cc); - ZERO_CONTENTS_N(curr_op, num_tables); + curr_op = 0; for (uint64 repeat_round = 0; repeat_round < 10; repeat_round++) { platform_default_log("Beginning repeat round %lu\n", repeat_round); @@ -1633,7 +1399,7 @@ test_splinter_periodic(system_config *cfg, total_time = platform_timestamp_elapsed(start_time); insert_latency_max = 0; - cache_io_stats(cc[0], &read_io_bytes, &write_io_bytes); + cache_io_stats(cc, &read_io_bytes, &write_io_bytes); io_mib = (read_io_bytes + write_io_bytes) / MiB; bandwidth = io_mib / NSEC_TO_SEC(total_time); @@ -1662,16 +1428,13 @@ test_splinter_periodic(system_config *cfg, NSEC_TO_MSEC(insert_latency_max)); } - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - cache_assert_free(spl->cc); - core_print_insertion_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - core_print_space_use(Platform_default_log_handle, spl); - cache_reset_stats(spl->cc); - } + cache_assert_free(spl->cc); + core_print_insertion_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + core_print_space_use(Platform_default_log_handle, spl); + cache_reset_stats(spl->cc); - ZERO_CONTENTS_N(curr_op, num_tables); + curr_op = 0; /* * ********** @@ -1684,10 +1447,7 @@ test_splinter_periodic(system_config *cfg, for (uint64 i = 0; i < num_lookup_threads; i++) { platform_status ret; - for (uint8 j = 0; j < num_tables; j++) { - async_ctxt_init( - hid, max_async_inflight, ¶ms[i].async_lookup[j]); - } + async_ctxt_init(hid, max_async_inflight, ¶ms[i].async_lookup); ret = platform_thread_create(¶ms[i].thread, FALSE, test_trunk_lookup_thread, @@ -1706,10 +1466,8 @@ test_splinter_periodic(system_config *cfg, uint64 num_async_lookups = 0; timestamp sync_lookup_latency_max = 0, async_lookup_latency_max = 0; for (uint64 i = 0; i < num_lookup_threads; i++) { - for (uint8 j = 0; j < num_tables; j++) { - async_ctxt_deinit(hid, params[i].async_lookup[j]); - params[i].async_lookup[j] = NULL; - } + async_ctxt_deinit(hid, params[i].async_lookup); + params[i].async_lookup = NULL; num_async_lookups += params[i].lookup_stats[ASYNC_LU].num_found + params[i].lookup_stats[ASYNC_LU].num_not_found; if (params[i].lookup_stats[SYNC_LU].latency_max @@ -1742,31 +1500,22 @@ test_splinter_periodic(system_config *cfg, platform_default_log("max lookup latency ns (sync=%lu, async=%lu)\n", sync_lookup_latency_max, async_lookup_latency_max); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - cache_assert_free(spl->cc); - core_print_lookup_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - cache_reset_stats(spl->cc); - } + cache_assert_free(spl->cc); + core_print_lookup_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + cache_reset_stats(spl->cc); } - uint64 total_ranges = 0; - - for (uint8 i = 0; i < num_tables; i++) { - per_table_ranges[i] = - ROUNDUP(per_table_inserts[i] / 128, TEST_RANGE_GRANULARITY); - total_ranges += per_table_ranges[i]; - } + uint64 total_ranges = ROUNDUP(total_inserts / 128, TEST_RANGE_GRANULARITY); for (uint64 i = 0; i < num_threads; i++) { - params[i].total_ops = per_table_ranges; + params[i].total_ops = total_ranges; params[i].op_granularity = TEST_RANGE_GRANULARITY; params[i].min_range_length = 1; params[i].max_range_length = 100; } - ZERO_CONTENTS_N(curr_op, num_tables); + curr_op = 0; if (num_range_threads != 0) { start_time = platform_get_timestamp(); @@ -1798,23 +1547,15 @@ test_splinter_periodic(system_config *cfg, total_time * num_range_threads / total_ranges); platform_default_log("splinter total range rate: %lu ops/second\n", SEC_TO_NSEC(total_ranges) / total_time); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - cache_assert_free(spl->cc); - core_print_lookup_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - cache_reset_stats(spl->cc); - } + cache_assert_free(spl->cc); + core_print_lookup_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + cache_reset_stats(spl->cc); - ZERO_CONTENTS_N(curr_op, num_tables); - total_ranges = 0; - for (uint8 i = 0; i < num_tables; i++) { - per_table_ranges[i] = - ROUNDUP(per_table_ranges[i] / 4, TEST_RANGE_GRANULARITY); - total_ranges += per_table_ranges[i]; - } + curr_op = 0; + total_ranges = ROUNDUP(total_ranges / 4, TEST_RANGE_GRANULARITY); for (uint64 i = 0; i < num_range_threads; i++) { - params[i].total_ops = per_table_ranges; + params[i].total_ops = total_ranges; params[i].op_granularity = TEST_RANGE_GRANULARITY; params[i].min_range_length = 512; params[i].max_range_length = 1024; @@ -1849,23 +1590,15 @@ test_splinter_periodic(system_config *cfg, total_time * num_range_threads / total_ranges); platform_default_log("splinter total range rate: %lu ops/second\n", SEC_TO_NSEC(total_ranges) / total_time); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - cache_assert_free(spl->cc); - core_print_lookup_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - cache_reset_stats(spl->cc); - } + cache_assert_free(spl->cc); + core_print_lookup_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + cache_reset_stats(spl->cc); - ZERO_CONTENTS_N(curr_op, num_tables); - total_ranges = 0; - for (uint8 i = 0; i < num_tables; i++) { - per_table_ranges[i] = - ROUNDUP(per_table_ranges[i] / 4, TEST_RANGE_GRANULARITY); - total_ranges += per_table_ranges[i]; - } + curr_op = 0; + total_ranges = ROUNDUP(total_ranges / 4, TEST_RANGE_GRANULARITY); for (uint64 i = 0; i < num_range_threads; i++) { - params[i].total_ops = per_table_ranges; + params[i].total_ops = total_ranges; params[i].op_granularity = TEST_RANGE_GRANULARITY; params[i].min_range_length = 131072 - 16384; params[i].max_range_length = 131072; @@ -1900,25 +1633,17 @@ test_splinter_periodic(system_config *cfg, total_time * num_range_threads / total_ranges); platform_default_log("splinter total range rate: %lu ops/second\n", SEC_TO_NSEC(total_ranges) / total_time); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - cache_assert_free(spl->cc); - core_print_lookup_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - cache_reset_stats(spl->cc); - } + cache_assert_free(spl->cc); + core_print_lookup_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + cache_reset_stats(spl->cc); } destroy_splinter: - test_trunk_destroy_tables(spl_tables, hid, num_tables); + test_trunk_destroy_table(spl, hid); platform_default_log("After destroy:\n"); - for (uint8 idx = 0; idx < num_caches; idx++) { - cache_print_stats(Platform_default_log_handle, cc[idx]); - } + cache_print_stats(Platform_default_log_handle, cc); platform_free(hid, params); - platform_free(hid, curr_op); - platform_free(hid, per_table_ranges); - platform_free(hid, per_table_inserts); return rc; } @@ -1938,7 +1663,7 @@ platform_status test_splinter_parallel_perf(system_config *cfg, test_config *test_cfg, allocator *al, - cache *cc[], + cache *cc, uint64 seed, const uint64 num_threads, const uint64 num_inserts_per_thread, @@ -1946,50 +1671,39 @@ test_splinter_parallel_perf(system_config *cfg, uint32 max_async_inflight, uint8 lookup_positive_pct, task_system *ts, - platform_heap_id hid, - uint8 num_tables, - uint8 num_caches) + platform_heap_id hid) { platform_assert((num_threads > 0), "num_threads=%lu", num_threads); platform_default_log( - "splinter_test: SplinterDB parallel performance test started with " - "%d tables\n", - num_tables); - core_handle *spl_tables; + "splinter_test: SplinterDB parallel performance test started\n"); + core_handle *spl; platform_status rc; platform_assert(num_inserts_per_thread <= num_lookups_per_thread); - rc = test_trunk_create_tables( - &spl_tables, cfg, al, cc, ts, hid, num_tables, num_caches); + rc = test_trunk_create_table(&spl, cfg, al, cc, ts, hid); if (!SUCCESS(rc)) { - platform_error_log("Failed to create splinter table(s): %s\n", + platform_error_log("Failed to create splinter table: %s\n", platform_status_to_string(rc)); return rc; } - uint64 *per_table_inserts = - TYPED_ARRAY_MALLOC(hid, per_table_inserts, num_tables); - uint64 *curr_insert_op = TYPED_ARRAY_ZALLOC(hid, curr_insert_op, num_tables); + uint64 curr_insert_op = 0; // This bit here onwards is very similar to splinter_perf_inserts(), but we // cannot directly call that function as the thread-handler for parallel- // performance exerciser is a different one; see below. - uint64 total_inserts = 0; - - total_inserts = - compute_per_table_inserts(per_table_inserts, cfg, test_cfg, num_tables); + uint64 total_inserts = compute_num_inserts(test_cfg); test_splinter_thread_params *params = TYPED_ARRAY_ZALLOC(hid, params, num_threads); // Load thread-specific exec-parameters for the insert workload load_thread_params(params, - spl_tables, + spl, test_cfg, - per_table_inserts, - curr_insert_op, - num_tables, + total_inserts, + &curr_insert_op, ts, (uint64)0, // unused num_insert_threads (uint64)0, // unused num_threads @@ -1999,14 +1713,13 @@ test_splinter_parallel_perf(system_config *cfg, // Load thread-specific execution parameters for the parallelism between // inserts & lookups in the workload for (uint64 i = 0; i < num_threads; i++) { - params[i].spl = spl_tables; + params[i].spl = spl; params[i].test_cfg = test_cfg; - params[i].total_ops = per_table_inserts; + params[i].total_ops = total_inserts; params[i].op_granularity = TEST_INSERT_GRANULARITY; params[i].thread_number = i; - params[i].num_tables = num_tables; params[i].ts = ts; - params[i].curr_op = curr_insert_op; + params[i].curr_op = &curr_insert_op; params[i].num_ops_per_thread[OP_INSERT] = num_inserts_per_thread; params[i].num_ops_per_thread[OP_LOOKUP] = num_lookups_per_thread; params[i].lookup_positive_pct = lookup_positive_pct; @@ -2015,8 +1728,7 @@ test_splinter_parallel_perf(system_config *cfg, uint64 start_time = platform_get_timestamp(); - do_n_async_ctxt_inits( - hid, num_threads, num_tables, max_async_inflight, cfg, params); + do_n_async_ctxt_inits(hid, num_threads, max_async_inflight, params); rc = do_n_thread_creates("insert/lookup thread", num_threads, @@ -2042,7 +1754,7 @@ test_splinter_parallel_perf(system_config *cfg, timestamp sync_lookup_latency_max = 0, async_lookup_latency_max = 0; uint64 total_insert_duration = 0; - do_n_async_ctxt_deinits(hid, num_threads, num_tables, params); + do_n_async_ctxt_deinits(hid, num_threads, params); for (uint64 i = 0; i < num_threads; i++) { insert_latency_max = @@ -2083,10 +1795,7 @@ test_splinter_parallel_perf(system_config *cfg, NSEC_TO_MSEC(insert_latency_max)); } - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - core_print_insertion_stats(Platform_default_log_handle, spl); - } + core_print_insertion_stats(Platform_default_log_handle, spl); if (num_threads > 0) { uint64 num_async_lookups = @@ -2111,23 +1820,16 @@ test_splinter_parallel_perf(system_config *cfg, platform_default_log("max lookup latency ns (sync=%lu, async=%lu)\n", sync_lookup_latency_max, async_lookup_latency_max); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - core_print_lookup_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - cache_reset_stats(spl->cc); - } + core_print_lookup_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + cache_reset_stats(spl->cc); } destroy_splinter: - test_trunk_destroy_tables(spl_tables, hid, num_tables); + test_trunk_destroy_table(spl, hid); platform_default_log("After destroy:\n"); - for (uint8 idx = 0; idx < num_caches; idx++) { - cache_print_stats(Platform_default_log_handle, cc[idx]); - } + cache_print_stats(Platform_default_log_handle, cc); platform_free(hid, params); - platform_free(hid, curr_insert_op); - platform_free(hid, per_table_inserts); return rc; } @@ -2135,44 +1837,31 @@ platform_status test_splinter_delete(system_config *cfg, test_config *test_cfg, allocator *al, - cache *cc[], + cache *cc, uint64 num_insert_threads, uint64 num_lookup_threads, uint32 max_async_inflight, task_system *ts, platform_heap_id hid, - uint8 num_tables, - uint8 num_caches, uint64 insert_rate) { - platform_default_log("splinter_test: SplinterDB deletion test started with " - "%d tables\n", - num_tables); - core_handle *spl_tables; + platform_default_log("splinter_test: SplinterDB deletion test started\n"); + core_handle *spl; platform_status rc; - rc = test_trunk_create_tables( - &spl_tables, cfg, al, cc, ts, hid, num_tables, num_caches); + rc = test_trunk_create_table(&spl, cfg, al, cc, ts, hid); if (!SUCCESS(rc)) { - platform_error_log("Failed to initialize splinter table(s): %s\n", + platform_error_log("Failed to initialize splinter table: %s\n", platform_status_to_string(rc)); return rc; } - uint64 tuple_size, num_inserts; - uint64 *per_table_inserts = - TYPED_ARRAY_MALLOC(hid, per_table_inserts, num_tables); - uint64 *curr_op = TYPED_ARRAY_ZALLOC(hid, curr_op, num_tables); - uint64 total_inserts = 0; - - for (uint8 i = 0; i < num_tables; i++) { - tuple_size = - test_cfg[i].key_size + generator_average_message_size(test_cfg->gen); - num_inserts = test_cfg[i].tree_size / tuple_size; - per_table_inserts[i] = ROUNDUP(num_inserts, TEST_INSERT_GRANULARITY); - total_inserts += per_table_inserts[i]; - } - uint64 num_threads = num_insert_threads; + uint64 curr_op = 0; + uint64 tuple_size = + test_cfg->key_size + generator_average_message_size(test_cfg->gen); + uint64 num_inserts = test_cfg->tree_size / tuple_size; + uint64 total_inserts = ROUNDUP(num_inserts, TEST_INSERT_GRANULARITY); + uint64 num_threads = num_insert_threads; if (num_lookup_threads > num_threads) { num_threads = num_lookup_threads; @@ -2183,13 +1872,12 @@ test_splinter_delete(system_config *cfg, ZERO_CONTENTS_N(params, num_threads); for (uint64 i = 0; i < num_threads; i++) { - params[i].spl = spl_tables; + params[i].spl = spl; params[i].test_cfg = test_cfg; - params[i].total_ops = per_table_inserts; - params[i].curr_op = curr_op; + params[i].total_ops = total_inserts; + params[i].curr_op = &curr_op; params[i].op_granularity = TEST_INSERT_GRANULARITY; params[i].thread_number = i; - params[i].num_tables = num_tables; params[i].ts = ts; params[i].insert_rate = insert_rate / num_insert_threads; } @@ -2217,11 +1905,8 @@ test_splinter_delete(system_config *cfg, "splinter total insertion rate: %lu insertions/second\n", SEC_TO_NSEC(total_inserts) / total_time); platform_default_log("After inserts:\n"); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - core_print_insertion_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - } + core_print_insertion_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); for (uint64 i = 0; i < num_insert_threads; i++) { if (!SUCCESS(params[i].rc)) { @@ -2234,7 +1919,7 @@ test_splinter_delete(system_config *cfg, // Deletes message_generate_set_message_type(test_cfg->gen, MESSAGE_TYPE_DELETE); - ZERO_CONTENTS_N(curr_op, num_tables); + curr_op = 0; start_time = platform_get_timestamp(); for (uint64 i = 0; i < num_insert_threads; i++) { platform_status ret; @@ -2254,11 +1939,8 @@ test_splinter_delete(system_config *cfg, platform_default_log("splinter total deletion rate: %lu insertions/second\n", SEC_TO_NSEC(total_inserts) / total_time); platform_default_log("After deletes:\n"); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - core_print_insertion_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - } + core_print_insertion_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); for (uint64 i = 0; i < num_insert_threads; i++) { if (!SUCCESS(params[i].rc)) { @@ -2273,13 +1955,11 @@ test_splinter_delete(system_config *cfg, for (uint64 i = 0; i < num_threads; i++) { params[i].expected_found = FALSE; } - ZERO_CONTENTS_N(curr_op, num_tables); + curr_op = 0; start_time = platform_get_timestamp(); for (uint64 i = 0; i < num_lookup_threads; i++) { - for (uint8 j = 0; j < num_tables; j++) { - async_ctxt_init(hid, max_async_inflight, ¶ms[i].async_lookup[j]); - } + async_ctxt_init(hid, max_async_inflight, ¶ms[i].async_lookup); rc = platform_thread_create( ¶ms[i].thread, FALSE, test_trunk_lookup_thread, ¶ms[i], hid); if (!SUCCESS(rc)) { @@ -2297,10 +1977,8 @@ test_splinter_delete(system_config *cfg, uint64 num_async_lookups = 0; for (uint64 i = 0; i < num_lookup_threads; i++) { - for (uint8 j = 0; j < num_tables; j++) { - async_ctxt_deinit(hid, params[i].async_lookup[j]); - params[i].async_lookup[j] = NULL; - } + async_ctxt_deinit(hid, params[i].async_lookup); + params[i].async_lookup = NULL; num_async_lookups += params[i].lookup_stats[ASYNC_LU].num_found + params[i].lookup_stats[ASYNC_LU].num_not_found; if (!SUCCESS(params[i].rc)) { @@ -2319,21 +1997,14 @@ test_splinter_delete(system_config *cfg, SEC_TO_NSEC(total_inserts) / total_time); platform_default_log("%lu%% lookups were async\n", num_async_lookups * 100 / total_inserts); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - core_print_lookup_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - } + core_print_lookup_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); destroy_splinter: - test_trunk_destroy_tables(spl_tables, hid, num_tables); + test_trunk_destroy_table(spl, hid); platform_default_log("After destroy:\n"); - for (uint8 idx = 0; idx < num_caches; idx++) { - cache_print_stats(Platform_default_log_handle, cc[idx]); - } + cache_print_stats(Platform_default_log_handle, cc); platform_free(hid, params); - platform_free(hid, curr_op); - platform_free(hid, per_table_inserts); return rc; } @@ -2355,7 +2026,7 @@ usage(const char *argv0) "--lookup-positive-percent [num] --seed [num]\n" "\t%s --functionality NUM_INSERTS CORRECTNESS_CHECK_FREQUENCY\n" "\t --max-async-inflight [num]\n" - "\t%s --num-tables [ --cache-per-table ] [ --use-shmem ]\n" + "\t%s [ --use-shmem ]\n" "\t%s --insert-rate (inserts_done_by_all_threads in a second)\n", argv0, argv0, @@ -2468,10 +2139,8 @@ splinter_test(int argc, char *argv[]) // Max async IOs inflight per-thread uint32 num_insert_threads, num_lookup_threads; uint32 num_range_lookup_threads, max_async_inflight; - uint32 num_pthreads = 0; - uint8 num_tables = 1; - bool32 cache_per_table = FALSE; - uint64 insert_rate = 0; // no rate throttling by default. + uint32 num_pthreads = 0; + uint64 insert_rate = 0; // no rate throttling by default. task_system ts; bool use_shmem = FALSE; uint8 lookup_positive_pct = 0; @@ -2567,30 +2236,6 @@ splinter_test(int argc, char *argv[]) * IF there are any more arguments remaining, parse them in sequence. * This set of args are expected to come in exactly this order. */ - if (config_argc > 0 - && strncmp(config_argv[0], "--num-tables", sizeof("--num-tables")) == 0) - { - if (config_argc < 2) { - usage(argv[0]); - return -1; - } - if (!try_string_to_uint8(config_argv[1], &num_tables)) { - usage(argv[0]); - return -1; - } - config_argc -= 2; - config_argv += 2; - } - - if (config_argc > 0 - && strncmp( - config_argv[0], "--cache-per-table", sizeof("--cache-per-table")) - == 0) - { - cache_per_table = TRUE; - config_argc -= 1; - config_argv += 1; - } if (config_argc > 0 && strncmp(config_argv[0], "--use-shmem", sizeof("--use-shmem")) == 0) { @@ -2631,14 +2276,6 @@ splinter_test(int argc, char *argv[]) config_argv += 2; } - /* - * Allocate 1 GiB for each cache, or 0.5 GiB for each splinter table, - * whichever is greater. - * Heap capacity should be within [2 * GiB, UINT32_MAX]. - */ - uint8 num_caches = cache_per_table ? num_tables : 1; - // uint64 heap_capacity = MAX(1024 * MiB * num_caches, 512 * MiB * - // num_tables); heap_capacity = MIN(heap_capacity, UINT32_MAX); uint64 heap_capacity = 512 * MiB; if (use_shmem) { platform_default_log( @@ -2656,17 +2293,12 @@ splinter_test(int argc, char *argv[]) * 2. Parse test_config options, see test_config_usage() */ - test_config *test_cfg = TYPED_ARRAY_ZALLOC(hid, test_cfg, num_tables); - for (uint8 i = 0; i < num_tables; i++) { - test_config_set_defaults(test, &test_cfg[i]); - - // All tables share the same message-generator. - test_cfg[i].gen = &gen; + test_config *test_cfg = TYPED_ZALLOC(hid, test_cfg); + test_config_set_defaults(test, test_cfg); + test_cfg->gen = &gen; + test_cfg->test_exec_cfg = &test_exec_cfg; - // And, all tests & tables share the same test-execution parameters. - test_cfg[i].test_exec_cfg = &test_exec_cfg; - } - rc = test_config_parse(test_cfg, num_tables, &config_argc, &config_argv); + rc = test_config_parse(test_cfg, 1, &config_argc, &config_argv); if (!SUCCESS(rc)) { platform_error_log("splinter_test: failed to parse config: %s\n", platform_status_to_string(rc)); @@ -2677,32 +2309,16 @@ splinter_test(int argc, char *argv[]) /* * 3. Parse trunk_config options, see config_usage() */ - system_config *system_cfg = TYPED_ARRAY_MALLOC(hid, system_cfg, num_tables); - test_workload_config *workload_cfg = - TYPED_ARRAY_MALLOC(hid, workload_cfg, num_tables); + system_config *system_cfg = TYPED_ZALLOC(hid, system_cfg); + test_workload_config *workload_cfg = TYPED_ZALLOC(hid, workload_cfg); rc = test_parse_args_n(system_cfg, &test_exec_cfg, workload_cfg, &gen, - num_tables, + 1, config_argc, config_argv); - - // if there are multiple cache capacity, cache_per_table needs to be TRUE - bool32 multi_cap = FALSE; - for (uint8 i = 0; i < num_tables; i++) { - if (system_cfg[i].cache_cfg.capacity != system_cfg[0].cache_cfg.capacity) - { - multi_cap = TRUE; - break; - } - } - if (multi_cap && !cache_per_table) { - platform_error_log("Multiple cache-capacity set but cache-per-table is " - "not set\n"); - rc = STATUS_BAD_PARAM; - } if (!SUCCESS(rc)) { platform_error_log("Failed to parse config arguments. See " "'driver_test %s --help' for usage information: %s\n", @@ -2711,9 +2327,7 @@ splinter_test(int argc, char *argv[]) goto cfg_free; } - for (uint8 i = 0; i < num_tables; i++) { - test_cfg[i].key_size = workload_cfg[i].key_size; - } + test_cfg->key_size = workload_cfg->key_size; seed = test_exec_cfg.seed; @@ -2722,26 +2336,26 @@ splinter_test(int argc, char *argv[]) MAX(num_lookup_threads, MAX(num_insert_threads, num_pthreads)); for (task_type type = 0; type != NUM_TASK_TYPES; type++) { - total_threads += system_cfg[0].task_cfg.num_background_threads[type]; + total_threads += system_cfg->task_cfg.num_background_threads[type]; } // Check if IO subsystem has enough reqs for max async IOs inflight - if (system_cfg[0].io_cfg.kernel_queue_size + if (system_cfg->io_cfg.kernel_queue_size < total_threads * max_async_inflight) { - system_cfg[0].io_cfg.kernel_queue_size = + system_cfg->io_cfg.kernel_queue_size = ROUNDUP(total_threads * max_async_inflight, 32); platform_default_log("Bumped up IO queue size to %lu\n", - system_cfg[0].io_cfg.kernel_queue_size); + system_cfg->io_cfg.kernel_queue_size); } - io_handle *io = io_handle_create(&system_cfg[0].io_cfg, hid); + io_handle *io = io_handle_create(&system_cfg->io_cfg, hid); if (io == NULL) { platform_error_log("Failed to create IO handle\n"); rc = STATUS_NO_MEMORY; goto cfg_free; } - rc = test_init_task_system(&ts, hid, &system_cfg[0].task_cfg); + rc = test_init_task_system(&ts, hid, &system_cfg->task_cfg); if (!SUCCESS(rc)) { platform_error_log("Failed to init splinter state: %s\n", platform_status_to_string(rc)); @@ -2750,44 +2364,34 @@ splinter_test(int argc, char *argv[]) rc_allocator al; rc_allocator_init( - &al, &system_cfg[0].allocator_cfg, io, hid, platform_get_module_id()); + &al, &system_cfg->allocator_cfg, io, hid, platform_get_module_id()); - platform_error_log("Running splinter_test with %d caches\n", num_caches); - clockcache *cc = TYPED_ARRAY_MALLOC(hid, cc, num_caches); + clockcache *cc = TYPED_ZALLOC(hid, cc); platform_assert(cc != NULL); - for (uint8 idx = 0; idx < num_caches; idx++) { - rc = clockcache_init(&cc[idx], - &system_cfg[idx].cache_cfg, - io, - (allocator *)&al, - "test", - hid, - platform_get_module_id()); - platform_assert_status_ok(rc); - } - allocator *alp = (allocator *)&al; + rc = clockcache_init(cc, + &system_cfg->cache_cfg, + io, + (allocator *)&al, + "test", + hid, + platform_get_module_id()); + platform_assert_status_ok(rc); - // Allocate an array of cache pointers to pass around. - cache **caches = TYPED_ARRAY_MALLOC(hid, caches, num_caches); - platform_assert(caches != NULL); - for (uint8 i = 0; i < num_caches; i++) { - caches[i] = (cache *)&cc[i]; - } + allocator *alp = (allocator *)&al; + cache *ccp = (cache *)cc; switch (test) { case perf: rc = test_splinter_perf(system_cfg, test_cfg, alp, - caches, + ccp, num_insert_threads, num_lookup_threads, num_range_lookup_threads, max_async_inflight, &ts, hid, - num_tables, - num_caches, insert_rate); platform_assert(SUCCESS(rc)); break; @@ -2795,14 +2399,12 @@ splinter_test(int argc, char *argv[]) rc = test_splinter_delete(system_cfg, test_cfg, alp, - caches, + ccp, num_insert_threads, num_lookup_threads, max_async_inflight, &ts, hid, - num_tables, - num_caches, insert_rate); platform_assert(SUCCESS(rc)); break; @@ -2810,15 +2412,13 @@ splinter_test(int argc, char *argv[]) rc = test_splinter_perf(system_cfg, test_cfg, alp, - caches, + ccp, num_insert_threads, num_lookup_threads, num_range_lookup_threads, max_async_inflight, &ts, hid, - num_tables, - num_caches, insert_rate); platform_assert(SUCCESS(rc)); break; @@ -2826,29 +2426,27 @@ splinter_test(int argc, char *argv[]) rc = test_splinter_perf(system_cfg, test_cfg, alp, - caches, + ccp, num_insert_threads, num_lookup_threads, num_range_lookup_threads, max_async_inflight, &ts, hid, - num_tables, - num_caches, insert_rate); platform_assert(SUCCESS(rc)); break; case parallel_perf: platform_assert( max_async_inflight == 0 - || (0 < system_cfg[0] - .task_cfg.num_background_threads[TASK_TYPE_MEMTABLE] - && 0 < system_cfg[0] - .task_cfg.num_background_threads[TASK_TYPE_NORMAL])); + || (0 < system_cfg->task_cfg + .num_background_threads[TASK_TYPE_MEMTABLE] + && 0 < system_cfg->task_cfg + .num_background_threads[TASK_TYPE_NORMAL])); rc = test_splinter_parallel_perf(system_cfg, test_cfg, alp, - caches, + ccp, seed, num_pthreads, 3, @@ -2856,35 +2454,29 @@ splinter_test(int argc, char *argv[]) max_async_inflight, lookup_positive_pct, &ts, - hid, - num_tables, - num_caches); + hid); platform_assert_status_ok(rc); break; case periodic: rc = test_splinter_periodic(system_cfg, test_cfg, alp, - caches, + ccp, num_insert_threads, num_lookup_threads, num_range_lookup_threads, max_async_inflight, &ts, hid, - num_tables, - num_caches, insert_rate); platform_assert(SUCCESS(rc)); break; case functionality: - for (uint8 i = 0; i < num_tables; i++) { - system_cfg[i].splinter_cfg.data_cfg->key_to_string = - test_data_config->key_to_string; - } + system_cfg->splinter_cfg.data_cfg->key_to_string = + test_data_config->key_to_string; rc = test_functionality(alp, io, - caches, + ccp, system_cfg, workload_cfg, seed, @@ -2892,8 +2484,6 @@ splinter_test(int argc, char *argv[]) correctness_check_frequency, &ts, hid, - num_tables, - num_caches, max_async_inflight); platform_assert_status_ok(rc); break; @@ -2910,10 +2500,7 @@ splinter_test(int argc, char *argv[]) platform_assert(0); } - for (uint8 idx = 0; idx < num_caches; idx++) { - clockcache_deinit(&cc[idx]); - } - platform_free(hid, caches); + clockcache_deinit(cc); platform_free(hid, cc); allocator_assert_noleaks(alp); rc_allocator_deinit(&al); diff --git a/tests/functional/test.h b/tests/functional/test.h index 82ab07ce..11bbd1bf 100644 --- a/tests/functional/test.h +++ b/tests/functional/test.h @@ -299,6 +299,11 @@ test_config_init(system_config *system_cfg, // OUT master_cfg->prefetch_budget, master_cfg->use_stats); + // 0 follows the cache size, mirroring the production default. + uint64 checkpoint_log_size = master_cfg->checkpoint_log_size != 0 + ? master_cfg->checkpoint_log_size + : master_cfg->cache_capacity; + rc = core_config_init(&system_cfg->splinter_cfg, &system_cfg->cache_cfg.super, system_cfg->data_cfg, @@ -308,6 +313,7 @@ test_config_init(system_config *system_cfg, // OUT master_cfg->queue_scale_percent, master_cfg->prefetch_budget, master_cfg->use_log, + checkpoint_log_size, master_cfg->use_stats, master_cfg->verbose_logging_enabled, master_cfg->log_handle); diff --git a/tests/functional/test_functionality.c b/tests/functional/test_functionality.c index 7cac864b..56f4ea37 100644 --- a/tests/functional/test_functionality.c +++ b/tests/functional/test_functionality.c @@ -633,14 +633,12 @@ cmp_ptrs(const void *a, const void *b) * where OP is insert, delete, increment, or decrement. * Verifies the results against the shadow. * - * The test does each of these operations for each of the "num_tables" passed - * in as argument. *----------------------------------------------------------------------------- */ platform_status test_functionality(allocator *al, io_handle *io, - cache *cc[], + cache *cc, system_config *cfg, test_workload_config *workload_cfg, uint64 seed, @@ -648,25 +646,16 @@ test_functionality(allocator *al, uint64 correctness_check_frequency, task_system *state, platform_heap_id hid, - uint8 num_tables, - uint8 num_caches, uint32 max_async_inflight) { - platform_error_log("Functional test started with %d tables\n", num_tables); + platform_error_log("Functional test started\n"); platform_assert(cc != NULL); - core_handle *spl_tables = TYPED_ARRAY_ZALLOC(hid, spl_tables, num_tables); - platform_assert(spl_tables != NULL); + core_handle *spl = TYPED_ZALLOC(hid, spl); + platform_assert(spl != NULL); - test_splinter_shadow_tree **shadows = - TYPED_ARRAY_ZALLOC(hid, shadows, num_tables); + test_splinter_shadow_tree *shadow = NULL; - platform_assert(shadows != NULL); - - allocator_root_id *splinters = - TYPED_ARRAY_ZALLOC(hid, splinters, num_tables); - - platform_assert(splinters != NULL); test_async_lookup *async_lookup; if (max_async_inflight > 0) { async_ctxt_init(hid, max_async_inflight, &async_lookup); @@ -679,49 +668,35 @@ test_functionality(allocator *al, random_init(&prg, seed, 0); - // Initialize the splinter/shadow for each splinter table. - for (uint8 idx = 0; idx < num_tables; idx++) { - cache *cache_to_use = num_caches > 1 ? cc[idx] : *cc; - status = test_splinter_shadow_create(&shadows[idx], hid, num_inserts); - if (!SUCCESS(status)) { - platform_error_log("Failed to init shadow for splinter: %s\n", - platform_status_to_string(status)); - goto cleanup; - } - splinters[idx] = test_generate_allocator_root_id(); - - status = core_mkfs(&spl_tables[idx], - &cfg[idx].splinter_cfg, - al, - cache_to_use, - state, - splinters[idx], - hid); - if (!SUCCESS(status)) { - platform_error_log("core_mkfs() failed for index=%d: %s\n", - idx, - platform_status_to_string(status)); - goto cleanup; - } + // Initialize the splinter table and its shadow. + status = test_splinter_shadow_create(&shadow, hid, num_inserts); + if (!SUCCESS(status)) { + platform_error_log("Failed to init shadow for splinter: %s\n", + platform_status_to_string(status)); + goto cleanup; } - // Validate each tree against an empty shadow. - for (uint8 idx = 0; idx < num_tables; idx++) { - core_handle *spl = &spl_tables[idx]; - test_splinter_shadow_tree *shadow = shadows[idx]; - status = validate_tree_against_shadow(spl, - &prg, - shadow, - hid, - workload_cfg[idx].key_size, - TRUE, - async_lookup); - if (!SUCCESS(status)) { - platform_error_log("Failed to validate empty tree against shadow: \ - %s\n", - platform_status_to_string(status)); - goto cleanup; - } + status = core_mkfs(spl, + &cfg->splinter_cfg, + al, + cc, + io, + state, + test_generate_allocator_root_id(), + hid); + if (!SUCCESS(status)) { + platform_error_log("core_mkfs() failed: %s\n", + platform_status_to_string(status)); + goto cleanup; + } + + // Validate the tree against an empty shadow. + status = validate_tree_against_shadow( + spl, &prg, shadow, hid, workload_cfg->key_size, TRUE, async_lookup); + if (!SUCCESS(status)) { + platform_error_log("Failed to validate empty tree against shadow: %s\n", + platform_status_to_string(status)); + goto cleanup; } // Run the test @@ -785,108 +760,69 @@ test_functionality(allocator *al, mindelta, maxdelta); - // Run the main test loop for each table. - for (uint8 idx = 0; idx < num_tables; idx++) { - // cache *cache_to_use = num_caches > 1 ? cc[idx] : *cc; - core_handle *spl = &spl_tables[idx]; - test_splinter_shadow_tree *shadow = shadows[idx]; - // allocator_root_id spl_id = splinters[idx]; - - status = insert_random_messages(spl, - shadow, - &prg, - workload_cfg[idx].key_size, - num_messages, - op, - minkey, - maxkey, - mindelta, - maxdelta); - if (!SUCCESS(status)) { - platform_error_log("Sumpin failed inserting messages: %s\n", - platform_status_to_string(status)); - goto cleanup; - } - - status = validate_tree_against_shadow( - spl, - &prg, - shadow, - hid, - workload_cfg[idx].key_size, - correctness_check_frequency - && (i % correctness_check_frequency) == 0, - async_lookup); - if (!SUCCESS(status)) { - platform_default_log("Failed to validate tree against shadow: %s\n", - platform_status_to_string(status)); - goto cleanup; - } - - /* if (correctness_check_frequency && i != 0 && */ - /* (i % correctness_check_frequency) == 0) { */ - /* platform_assert(trunk_verify_tree(spl)); */ - /* platform_default_log("Dismount and remount\n"); */ - /* allocator_config *al_cfg = ((rc_allocator *)al)->cfg; */ - /* uint64 prev_root_addr = spl->root_addr; */ - /* trunk_dismount(spl); */ - /* rc_allocator_dismount((rc_allocator *)al); */ - /* rc_allocator_mount((rc_allocator *)al, al_cfg, io, hh, hid, */ - /* platform_get_module_id()); */ - /* spl = trunk_mount(&cfg[idx], al, cache_to_use, state, spl_id, - */ - /* hid); */ - /* spl_tables[idx] = spl; */ - /* if (spl->root_addr != prev_root_addr) { */ - /* platform_error_log("Mismatch in root addr across mount\n"); - */ - /* status = STATUS_TEST_FAILED; */ - /* goto cleanup; */ - /* } */ - /* } */ + status = insert_random_messages(spl, + shadow, + &prg, + workload_cfg->key_size, + num_messages, + op, + minkey, + maxkey, + mindelta, + maxdelta); + if (!SUCCESS(status)) { + platform_error_log("Sumpin failed inserting messages: %s\n", + platform_status_to_string(status)); + goto cleanup; } - total_inserts += num_messages; - i++; - } - - // Validate each tree against the shadow one last time. - for (uint8 idx = 0; idx < num_tables; idx++) { - core_handle *spl = &spl_tables[idx]; - test_splinter_shadow_tree *shadow = shadows[idx]; - status = validate_tree_against_shadow( spl, &prg, shadow, hid, - workload_cfg[idx].key_size, - correctness_check_frequency - && ((i - 1) % correctness_check_frequency) != 0, + workload_cfg->key_size, + correctness_check_frequency && (i % correctness_check_frequency) == 0, async_lookup); if (!SUCCESS(status)) { - platform_error_log("Failed to validate tree against shadow one \ - last time: %s\n", - platform_status_to_string(status)); + platform_default_log("Failed to validate tree against shadow: %s\n", + platform_status_to_string(status)); goto cleanup; } + + total_inserts += num_messages; + i++; + } + + // Validate the tree against the shadow one last time. + status = validate_tree_against_shadow( + spl, + &prg, + shadow, + hid, + workload_cfg->key_size, + correctness_check_frequency + && ((i - 1) % correctness_check_frequency) != 0, + async_lookup); + if (!SUCCESS(status)) { + platform_error_log("Failed to validate tree against shadow one last " + "time: %s\n", + platform_status_to_string(status)); + goto cleanup; } cleanup: - for (uint8 idx = 0; idx < num_tables; idx++) { - if (spl_tables[idx].cc != NULL) { - core_destroy(&spl_tables[idx]); - } - if (shadows[idx] != NULL) { - test_splinter_shadow_destroy(hid, shadows[idx]); - } + if (spl->cc != NULL) { + core_print_insertion_stats(Platform_default_log_handle, spl); + core_destroy(spl); + } + if (shadow != NULL) { + test_splinter_shadow_destroy(hid, shadow); } if (async_lookup) { async_ctxt_deinit(hid, async_lookup); } - platform_free(hid, spl_tables); - platform_free(hid, splinters); - platform_free(hid, shadows); + platform_free(hid, spl); return status; } diff --git a/tests/functional/test_functionality.h b/tests/functional/test_functionality.h index bbbe1a42..ddd51ad5 100644 --- a/tests/functional/test_functionality.h +++ b/tests/functional/test_functionality.h @@ -10,7 +10,7 @@ platform_status test_functionality(allocator *al, io_handle *io, - cache *cc[], + cache *cc, system_config *cfg, test_workload_config *workload_cfg, uint64 seed, @@ -18,6 +18,4 @@ test_functionality(allocator *al, uint64 correctness_check_frequency, task_system *ts, platform_heap_id hid, - uint8 num_tables, - uint8 num_caches, uint32 max_async_inflight); diff --git a/tests/functional/ycsb_test.c b/tests/functional/ycsb_test.c index 244f8300..7256f8ac 100644 --- a/tests/functional/ycsb_test.c +++ b/tests/functional/ycsb_test.c @@ -1295,6 +1295,7 @@ ycsb_test(int argc, char *argv[]) &system_cfg->splinter_cfg, (allocator *)&al, (cache *)cc, + io, &ts, test_generate_allocator_root_id(), hid); @@ -1314,6 +1315,7 @@ ycsb_test(int argc, char *argv[]) &system_cfg->splinter_cfg, (allocator *)&al, (cache *)cc, + io, &ts, test_generate_allocator_root_id(), hid); @@ -1325,7 +1327,8 @@ ycsb_test(int argc, char *argv[]) core_unmount(&spl); clockcache_deinit(cc); platform_free(hid, cc); - rc_allocator_unmount(&al); + // core_unmount() already persisted the map and published the superblock. + rc_allocator_deinit(&al); test_deinit_task_system(&ts); rc = STATUS_OK; diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index 5377cde9..a3301f89 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -84,7 +84,6 @@ CTEST_DATA(splinter) uint32 num_insert_threads; uint32 num_lookup_threads; uint32 max_async_inflight; - int spl_num_tables; rc_allocator al; @@ -115,12 +114,10 @@ CTEST_SETUP(splinter) data->num_insert_threads = 1; data->num_lookup_threads = 1; data->max_async_inflight = 64; - data->spl_num_tables = 1; - bool32 cache_per_table = FALSE; - int num_tables = data->spl_num_tables; // Cache, for re-use below - uint8 num_caches = (cache_per_table ? num_tables : 1); - uint64 heap_capacity = 512 * MiB; + // The config layer still parses per-config arrays; this test uses one. + int num_tables = 1; + uint64 heap_capacity = 1024 * MiB; // Create a heap for io, allocator, cache and splinter platform_status rc = platform_heap_create(platform_get_module_id(), @@ -173,20 +170,18 @@ CTEST_SETUP(splinter) rc_allocator_init(&data->al, &data->system_cfg->allocator_cfg, data->io, data->hid, platform_get_module_id()); - data->clock_cache = TYPED_ARRAY_MALLOC(data->hid, data->clock_cache, num_caches); + data->clock_cache = TYPED_MALLOC(data->hid, data->clock_cache); ASSERT_TRUE((data->clock_cache != NULL)); - for (uint8 idx = 0; idx < num_caches; idx++) { - rc = clockcache_init(&data->clock_cache[idx], - &data->system_cfg[idx].cache_cfg, - data->io, - (allocator *)&data->al, - "test", - data->hid, - platform_get_module_id()); + rc = clockcache_init(data->clock_cache, + &data->system_cfg->cache_cfg, + data->io, + (allocator *)&data->al, + "test", + data->hid, + platform_get_module_id()); - ASSERT_TRUE(SUCCESS(rc), "clockcache_init() failed for index=%d. ", idx); - } + ASSERT_TRUE(SUCCESS(rc), "clockcache_init() failed. "); } // clang-format on @@ -235,6 +230,7 @@ CTEST2(splinter, test_inserts) &data->system_cfg->splinter_cfg, alp, (cache *)data->clock_cache, + data->io, &data->tasks, test_generate_allocator_root_id(), data->hid); @@ -250,6 +246,404 @@ CTEST2(splinter, test_inserts) core_destroy(&spl); } +/* + * With logging enabled, core_checkpoint() rotates the log via the two-log + * protocol and advances the durable root. Data inserted before the checkpoint + * must survive it, and teardown's allocator_assert_noleaks() must pass (the + * sealed log's extents are freed, the root is not leaked or double-freed). + * + * A zero rotation timeout forces the rotation immediately rather than waiting + * for insert traffic to trigger one. The second checkpoint takes no new + * inserts, so it also covers the empty-memtable forced rotation: the resulting + * generation retires with no branch at all. + */ +CTEST2(splinter, test_two_log_checkpoint) +{ + allocator *alp = (allocator *)&data->al; + data->system_cfg->splinter_cfg.use_log = + TRUE; // exercise the two-log lifecycle + + core_handle spl; + platform_status rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + test_generate_allocator_root_id(), + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + uint64 num_inserts = splinter_do_inserts(data, &spl, FALSE, NULL); + ASSERT_NOT_EQUAL(0, num_inserts); + + // Checkpoint: seal the live log into the sealed slot, start a fresh live + // log, incorporate, advance the durable root, then clear + free the sealed + // log. + rc = core_checkpoint(&spl, 0); + ASSERT_TRUE(SUCCESS(rc)); + + // A sample of keys must still be found after the checkpoint. + lookup_result qdata; + lookup_result_init( + &qdata, spl.cfg.data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + const size_t key_size = data->workload_cfg->key_size; + uint64 verify_count = (num_inserts < 1000) ? num_inserts : 1000; + for (uint64 i = 0; i < verify_count; i++) { + test_key(&keybuf, TEST_RANDOM, i, 0, 0, key_size, 0); + rc = core_lookup(&spl, key_buffer_key(&keybuf), &qdata); + ASSERT_TRUE(SUCCESS(rc)); + verify_tuple( + &spl, + &data->gen, + i, + key_buffer_key(&keybuf), + merge_accumulator_to_message(lookup_result_accumulator(&qdata)), + TRUE); + } + lookup_result_deinit(&qdata); + + // Second checkpoint with no new inserts is a clean rotate. + rc = core_checkpoint(&spl, 0); + ASSERT_TRUE(SUCCESS(rc)); + + core_destroy(&spl); +} + +/* + * An application that manages checkpoints itself: the interval policy is off, + * so nothing arms a checkpoint automatically and core_checkpoint() is the only + * thing that can rotate the log. It must therefore cut the log and reclaim + * what it retires, or the live log would grow without bound. + * + * Each checkpoint must (a) install a different live log -- proving a cut + * happened -- and (b) leave the retired log's metadata extent free. Total + * space in use must also stay flat across many checkpoints of the same data. + */ +CTEST2(splinter, test_self_managed_checkpoints_reclaim_log_space) +{ + allocator *alp = (allocator *)&data->al; + data->system_cfg->splinter_cfg.use_log = TRUE; + // No automatic checkpoints: this test drives them all itself. + data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = 0; + + core_handle spl; + platform_status rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + test_generate_allocator_root_id(), + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + uint64 num_inserts = splinter_do_inserts(data, &spl, FALSE, NULL); + ASSERT_NOT_EQUAL(0, num_inserts); + + superblock_tree_record rec; + const uint64 num_checkpoints = 10; + uint64 baseline_in_use = 0; + + for (uint64 i = 0; i < num_checkpoints; i++) { + superblock_get_tree_record(&spl.superblock, &rec); + uint64 retired_meta_addr = rec.live_log.meta_addr; + ASSERT_NOT_EQUAL(0, retired_meta_addr); + + rc = core_checkpoint(&spl, 0); + ASSERT_TRUE(SUCCESS(rc)); + + // (a) A cut happened: a different log is now live, and it covers only + // generations from the cut onward. + superblock_get_tree_record(&spl.superblock, &rec); + ASSERT_NOT_EQUAL(retired_meta_addr, rec.live_log.meta_addr); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(rec.sealed_log)); + + // (b) The retired log's space came back. + ASSERT_EQUAL(0, + allocator_get_refcount(alp, retired_meta_addr), + "checkpoint %lu did not reclaim retired log at %lu\n", + i, + retired_meta_addr); + + // Space in use must not creep upward checkpoint over checkpoint. + if (i == 0) { + baseline_in_use = allocator_in_use(alp); + } else { + ASSERT_TRUE(allocator_in_use(alp) <= baseline_in_use, + "space in use grew from %lu to %lu by checkpoint %lu\n", + baseline_in_use, + allocator_in_use(alp), + i); + } + } + + core_destroy(&spl); +} + +/* + * Shared workload for the automatic-checkpoint tests: create with auto + * checkpoints enabled, insert enough to drive several rotations, verify a + * sample of keys survives the mid-run log rotations, then destroy. The + * teardown's allocator_assert_noleaks() must pass -- each sealed log's extents + * are freed as its checkpoint completes, and the root is neither leaked nor + * double-freed. The caller sets up the task system (foreground or background). + */ +static void +run_auto_checkpoint_workload(void *datap, uint64 log_size_threshold) +{ + struct CTEST_IMPL_DATA_SNAME(splinter) *data = + (struct CTEST_IMPL_DATA_SNAME(splinter) *)datap; + + allocator *alp = (allocator *)&data->al; + data->system_cfg->splinter_cfg.use_log = TRUE; + // Rotate the log / advance the durable root once the log reaches this size. + data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = + log_size_threshold; + + core_handle spl; + platform_status rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + test_generate_allocator_root_id(), + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + uint64 num_inserts = splinter_do_inserts(data, &spl, FALSE, NULL); + ASSERT_NOT_EQUAL(0, num_inserts); + + // Drain so any in-flight checkpoint completes and the state settles. + rc = task_perform_until_quiescent(spl.ts); + ASSERT_TRUE(SUCCESS(rc)); + + if (log_size_threshold != 0) { + // The inserts must have driven at least one automatic checkpoint through + // to completion. + ASSERT_NOT_EQUAL(0, spl.checkpoint.completions); + + // At rest a checkpoint has either completed (IDLE) or been armed for a + // rotation that idle never triggered (PENDING); both leave no sealed log. + ASSERT_TRUE(spl.checkpoint.phase == CORE_CHECKPOINT_IDLE + || spl.checkpoint.phase == CORE_CHECKPOINT_PENDING); + superblock_tree_record rec; + superblock_get_tree_record(&spl.superblock, &rec); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(rec.sealed_log)); + // A checkpoint published an advanced, incorporated durable root mid-run: + // at least one generation was folded in, so the first unincorporated + // generation has advanced past 0. + ASSERT_NOT_EQUAL(0, rec.first_unincorporated_generation); + } + + // A sample of keys must still be found after the rotations. + lookup_result qdata; + lookup_result_init( + &qdata, spl.cfg.data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + const size_t key_size = data->workload_cfg->key_size; + uint64 verify_count = (num_inserts < 1000) ? num_inserts : 1000; + for (uint64 i = 0; i < verify_count; i++) { + test_key(&keybuf, TEST_RANDOM, i, 0, 0, key_size, 0); + rc = core_lookup(&spl, key_buffer_key(&keybuf), &qdata); + ASSERT_TRUE(SUCCESS(rc)); + verify_tuple( + &spl, + &data->gen, + i, + key_buffer_key(&keybuf), + merge_accumulator_to_message(lookup_result_accumulator(&qdata)), + TRUE); + } + lookup_result_deinit(&qdata); + + core_destroy(&spl); +} + +/* + * Automatic, incorporation-driven checkpoints (foreground): the log is rotated + * and the durable root advanced during normal inserts, with no stop-the-world. + */ +CTEST2(splinter, test_auto_checkpoint) +{ + // One extent: small enough that the test workload drives several rotations. + run_auto_checkpoint_workload(data, 2 * data->system_cfg->io_cfg.extent_size); +} + +/* + * The reason the policy is sized in log bytes rather than memtable generations. + * + * Repeatedly overwriting one key updates the memtable btree in place, so it + * never accumulates extents, never becomes "full", and never rotates -- the + * generation stays put for the whole workload. Every write still appends to + * the log, so the log grows without bound. A generation-based trigger could + * never fire here; the size-based one must. + */ +CTEST2(splinter, test_auto_checkpoint_on_overwrites) +{ + allocator *alp = (allocator *)&data->al; + data->system_cfg->splinter_cfg.use_log = TRUE; + data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = + 2 * data->system_cfg->io_cfg.extent_size; + // Also verify the reported checkpoint count against the internal one. + data->system_cfg->splinter_cfg.use_stats = TRUE; + + core_handle spl; + platform_status rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + test_generate_allocator_root_id(), + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + uint64 start_generation = memtable_generation(&spl.mt_ctxt); + + // Hammer a single key. Enough writes to push the log well past one extent. + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + merge_accumulator msg; + merge_accumulator_init(&msg, data->hid); + const uint64 num_overwrites = 20000; + for (uint64 i = 0; i < num_overwrites; i++) { + test_key(&keybuf, TEST_RANDOM, 0, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, i, &msg); + rc = core_insert(&spl, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); + ASSERT_TRUE(SUCCESS(rc)); + } + + rc = task_perform_until_quiescent(spl.ts); + ASSERT_TRUE(SUCCESS(rc)); + + /* + * The workload never filled a memtable on its own, so a generation-based + * policy would have had nothing to trigger on: any generation advance here + * came from a checkpoint forcing a rotation, not from the memtable filling. + */ + ASSERT_NOT_EQUAL(0, + spl.checkpoint.completions, + "overwrite-only workload did not trigger a checkpoint; " + "generation went %lu -> %lu\n", + start_generation, + memtable_generation(&spl.mt_ctxt)); + + /* + * The reported statistic must agree with the machinery's own count: it is + * summed across threads, so this catches both a missed increment and a + * double count. + */ + uint64 reported = 0; + for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { + reported += spl.stats[thr_i].checkpoints_completed; + } + ASSERT_EQUAL(spl.checkpoint.completions, + reported, + "checkpoints_completed stat (%lu) disagrees with the " + "checkpoint state's count (%lu)\n", + reported, + spl.checkpoint.completions); + + // The surviving value must be the last one written. + lookup_result qdata; + lookup_result_init( + &qdata, spl.cfg.data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + test_key(&keybuf, TEST_RANDOM, 0, 0, 0, data->workload_cfg->key_size, 0); + rc = core_lookup(&spl, key_buffer_key(&keybuf), &qdata); + ASSERT_TRUE(SUCCESS(rc)); + generate_test_message(&data->gen, num_overwrites - 1, &msg); + ASSERT_EQUAL(0, + message_lex_cmp(merge_accumulator_to_message(&msg), + merge_accumulator_to_message( + lookup_result_accumulator(&qdata)))); + lookup_result_deinit(&qdata); + merge_accumulator_deinit(&msg); + + core_destroy(&spl); +} + +/* + * The second checkpoint slot is a torn-write fallback, not permission for a + * normal mount to silently roll back past a newer, valid active record. A + * successful mount publishes such an active record; until crash recovery is + * implemented, a concurrent/restarted normal mount must reject it even + * though the preceding clean record remains valid in the other slot. + */ +CTEST2(splinter, test_mount_rejects_newer_active_checkpoint) +{ + allocator *alp = (allocator *)&data->al; + allocator_root_id root_id = test_generate_allocator_root_id(); + core_handle created, mounted, rejected, cleanup; + platform_status rc; + + rc = core_mkfs(&created, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + /* Give the clean record a real COW root, not just the empty-tree root. */ + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + merge_accumulator msg; + merge_accumulator_init(&msg, data->hid); + test_key(&keybuf, TEST_RANDOM, 1, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, 1, &msg); + rc = core_insert(&created, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); + merge_accumulator_deinit(&msg); + ASSERT_TRUE(SUCCESS(rc)); + + rc = core_unmount(&created); + ASSERT_TRUE(SUCCESS(rc)); + + /* This mount advances the A/B sequence with an unmounted=FALSE record. */ + rc = core_mount(&mounted, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + rc = core_mount(&rejected, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(STATUS_IS_EQ(rc, STATUS_INVALID_STATE)); + + /* Finish cleanly, then prove the same record pair is mountable again. */ + rc = core_unmount(&mounted); + ASSERT_TRUE(SUCCESS(rc)); + + rc = core_mount(&cleanup, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + core_destroy(&cleanup); +} + static void trunk_shadow_init(trunk_shadow *shadow, data_config *data_cfg, @@ -409,6 +803,7 @@ CTEST2(splinter, test_lookups) &data->system_cfg->splinter_cfg, alp, (cache *)data->clock_cache, + data->io, &data->tasks, test_generate_allocator_root_id(), data->hid); @@ -634,6 +1029,7 @@ CTEST2(splinter, test_splinter_print_diags) &data->system_cfg->splinter_cfg, alp, (cache *)data->clock_cache, + data->io, &data->tasks, test_generate_allocator_root_id(), data->hid); diff --git a/tests/unit/splinterdb_optimize_test.c b/tests/unit/splinterdb_optimize_test.c index a8587cf6..215e8219 100644 --- a/tests/unit/splinterdb_optimize_test.c +++ b/tests/unit/splinterdb_optimize_test.c @@ -284,9 +284,8 @@ create_optimize_cfg(splinterdb_config *out_cfg, data_config *default_data_cfg) static void force_flush_current_memtable(splinterdb *kvsb) { - core_handle *core = (core_handle *)splinterdb_get_trunk_handle(kvsb); - uint64 generation = memtable_force_finalize(&core->mt_ctxt); - core->mt_ctxt.process(core->mt_ctxt.process_ctxt, generation); + core_handle *core = (core_handle *)splinterdb_get_trunk_handle(kvsb); + memtable_force_rotation(&core->mt_ctxt); platform_status rc = task_perform_until_quiescent(core->ts); ASSERT_TRUE(SUCCESS(rc)); } diff --git a/tests/unit/splinterdb_quick_test.c b/tests/unit/splinterdb_quick_test.c index 5dee4428..4eb29e23 100644 --- a/tests/unit/splinterdb_quick_test.c +++ b/tests/unit/splinterdb_quick_test.c @@ -1251,6 +1251,49 @@ CTEST2(splinterdb_quick, test_close_and_reopen) splinterdb_lookup_result_deinit(&result); } +/* + * With logging enabled, the two-log lifecycle records the live log in the + * superblock at create/mount and discards it (seal + dec_ref) at a clean close. + * A reopen must still find data inserted before the close. + */ +CTEST2(splinterdb_quick, test_logged_close_and_reopen) +{ + // Re-create the instance with logging enabled (SETUP created it without). + splinterdb_close(&data->kvsb); + data->cfg.use_log = TRUE; + int rc = splinterdb_create(&data->cfg, &data->kvsb); + ASSERT_EQUAL(0, rc); + + slice user_key = slice_create(strlen("logged-key"), "logged-key"); + const char *val = "logged-value"; + const size_t val_len = strlen(val); + rc = + splinterdb_insert(data->kvsb, user_key, slice_create(val_len, val), NULL); + ASSERT_EQUAL(0, rc); + + splinterdb_close(&data->kvsb); + rc = splinterdb_open(&data->cfg, &data->kvsb); + ASSERT_EQUAL(0, rc); + + splinterdb_lookup_result result; + splinterdb_lookup_result_init( + data->kvsb, &result, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + rc = splinterdb_lookup(data->kvsb, user_key, &result); + ASSERT_EQUAL(0, rc); + ASSERT_TRUE(splinterdb_lookup_found(&result)); + + slice value; + rc = splinterdb_lookup_result_value(&result, &value); + ASSERT_EQUAL(0, rc); + ASSERT_EQUAL(val_len, slice_length(value)); + ASSERT_STREQN(val, + slice_data(value), + slice_length(value), + "logged reopen value mismatch up to %d bytes\n", + val_len); + splinterdb_lookup_result_deinit(&result); +} + /* * Regression test for bug where repeating a cycle of insert-close-reopen * causes a space leak and eventually hits an assertion @@ -1766,10 +1809,9 @@ create_default_cfg(splinterdb_config *out_cfg, data_config *default_data_cfg) static uint64 force_flush_current_memtable(splinterdb *kvsb) { - core_handle *core = (core_handle *)splinterdb_get_trunk_handle(kvsb); - uint64 generation = memtable_force_finalize(&core->mt_ctxt); - core->mt_ctxt.process(core->mt_ctxt.process_ctxt, generation); - platform_status rc = task_perform_until_quiescent(core->ts); + core_handle *core = (core_handle *)splinterdb_get_trunk_handle(kvsb); + uint64 generation = memtable_force_rotation(&core->mt_ctxt); + platform_status rc = task_perform_until_quiescent(core->ts); ASSERT_TRUE(SUCCESS(rc)); return generation; } diff --git a/tests/unit/superblock_test.c b/tests/unit/superblock_test.c new file mode 100644 index 00000000..b4f49324 --- /dev/null +++ b/tests/unit/superblock_test.c @@ -0,0 +1,407 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +/* + * ----------------------------------------------------------------------------- + * superblock_test.c -- + * + * Unit tests for the unified superblock (src/superblock.c): fresh format + * state, durable publish + re-mount round-trip, the A/B generation selection, + * and torn-write fallback (a corrupted newest slot must leave the previous + * generation intact and mountable). + * ----------------------------------------------------------------------------- + */ +#include "unit_tests.h" +#include "ctest.h" // This is required for all test-case files. +#include "platform.h" +#include "config.h" // Reqd for definition of master_config{} +#include "allocator.h" +#include "superblock.h" + +CTEST_DATA(superblock) +{ + platform_heap_id hid; + io_config io_cfg; + io_handle *ioh; + allocator_config allocator_cfg; +}; + +CTEST_SETUP(superblock) +{ + platform_register_thread(); + + bool use_shmem = config_parse_use_shmem(Ctest_argc, (char **)Ctest_argv); + + platform_status rc = platform_heap_create( + platform_get_module_id(), (256 * MiB), use_shmem, &data->hid); + platform_assert_status_ok(rc); + + master_config master_cfg; + config_set_defaults(&master_cfg); + + io_config_init(&data->io_cfg, + master_cfg.page_size, + master_cfg.extent_size, + master_cfg.io_flags, + master_cfg.io_perms, + master_cfg.io_async_queue_depth, + master_cfg.io_filename); + + data->ioh = io_handle_create(&data->io_cfg, data->hid); + ASSERT_TRUE(data->ioh != NULL, "Failed to create IO handle\n"); + + allocator_config_init( + &data->allocator_cfg, &data->io_cfg, master_cfg.allocator_capacity); +} + +CTEST_TEARDOWN(superblock) +{ + io_handle_destroy(data->ioh); + platform_heap_destroy(&data->hid); + platform_deregister_thread(); +} + +/* Zero one raw superblock slot (page) and make the corruption durable. */ +static void +superblock_test_corrupt_slot(io_handle *io, uint64 page_size, uint64 slot) +{ + buffer_handle buffer; + platform_status rc = platform_buffer_init(&buffer, page_size); + platform_assert_status_ok(rc); + + void *page = platform_buffer_getaddr(&buffer); + memset(page, 0, page_size); + rc = io_write(io, page, page_size, slot * page_size); + platform_assert_status_ok(rc); + rc = io_durable_barrier(io); + platform_assert_status_ok(rc); + + platform_buffer_deinit(&buffer); +} + +/* + * A freshly formatted superblock has an empty tree table and invalid (rebuild) + * allocation state, and a subsequent mount reads that back. + */ +CTEST2(superblock, test_format_sets_fresh_state) +{ + superblock_context ctx; + platform_status rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + rc = superblock_format(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_FALSE(superblock_allocation_state_valid(&ctx)); + superblock_tree_record rec; + superblock_get_tree_record(&ctx, &rec); + ASSERT_EQUAL(0, rec.root_addr); // empty tree + ASSERT_EQUAL(0, + rec.first_unincorporated_generation); // replay from the start + superblock_context_deinit(&ctx); + + rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_mount(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_FALSE(superblock_allocation_state_valid(&ctx)); + superblock_get_tree_record(&ctx, &rec); + ASSERT_EQUAL(0, rec.root_addr); + superblock_context_deinit(&ctx); +} + +/* + * A durable tree snapshot and a subsequent durable allocator snapshot (the + * clean-unmount Part A / Part B sequence) survive a deinit + re-mount: the + * newest generation wins. + */ +CTEST2(superblock, test_snapshot_persists_state) +{ + superblock_context ctx; + platform_status rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_format(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + + superblock_log_head live = { + .addr = 0x6000, .meta_addr = 0x8000, .magic = 0x11}; + superblock_log_cut(&ctx, live); + superblock_snapshot_tree(&ctx, 0x4000, 0); + rc = superblock_make_durable(&ctx); + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_FALSE( + superblock_allocation_state_valid(&ctx)); // snapshot invalidated + + superblock_snapshot_allocator(&ctx, 0x8000); + rc = superblock_make_durable(&ctx); + ASSERT_TRUE(SUCCESS(rc)); + superblock_context_deinit(&ctx); + + rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_mount(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_TRUE(superblock_allocation_state_valid(&ctx)); + ASSERT_EQUAL(0x8000, superblock_allocation_state_addr(&ctx)); + + superblock_tree_record got; + superblock_get_tree_record(&ctx, &got); + ASSERT_EQUAL(0x4000, got.root_addr); + ASSERT_EQUAL(0x6000, got.live_log.addr); + ASSERT_EQUAL(0x8000, got.live_log.meta_addr); + ASSERT_EQUAL(0x11, got.live_log.magic); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(got.sealed_log)); // no checkpoint in progress + superblock_context_deinit(&ctx); +} + +/* + * Steady, begin-checkpoint, and complete-checkpoint tree-record states. L1 + * covers generations 0..5 and is cut at 5, so L2 takes over at 6. + */ +static const superblock_log_head TEST_LOG_L1 = {.addr = 0x6000, + .meta_addr = 0x8000, + .magic = 0x11, + .start_generation = 0}; +static const superblock_log_head TEST_LOG_L2 = {.addr = 0x10000, + .meta_addr = 0x12000, + .magic = 0x22, + .start_generation = 6}; + +/* + * Walk the two-log checkpoint state machine through the superblock and confirm + * each published state reads back: steady {root R0, live L1, no sealed} -> + * begin {root R0, sealed L1, live L2} -> complete {root R1, live L2, no + * sealed}. + */ +CTEST2(superblock, test_two_log_checkpoint_transitions) +{ + superblock_context ctx; + platform_status rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_format(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + + // Steady: root R0, live L1, no sealed log. + superblock_log_cut(&ctx, TEST_LOG_L1); + superblock_snapshot_tree(&ctx, 0x4000, 5); + rc = superblock_make_durable(&ctx); + ASSERT_TRUE(SUCCESS(rc)); + + // Begin: cut the log -- L1 becomes sealed, L2 becomes live. + superblock_log_cut(&ctx, TEST_LOG_L2); + rc = superblock_make_durable(&ctx); + ASSERT_TRUE(SUCCESS(rc)); + + superblock_tree_record got; + superblock_get_tree_record(&ctx, &got); + ASSERT_EQUAL(TEST_LOG_L1.meta_addr, got.sealed_log.meta_addr); + ASSERT_EQUAL(TEST_LOG_L2.meta_addr, got.live_log.meta_addr); + + // Complete: advance the root past L1's coverage (first unincorporated 9 >= + // L2's start 6), so the sealed log is dropped and L2 carries forward. + superblock_snapshot_tree(&ctx, 0x4400, 9); + rc = superblock_make_durable(&ctx); + ASSERT_TRUE(SUCCESS(rc)); + superblock_context_deinit(&ctx); + + // A fresh mount reads the completed state. + rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_mount(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + superblock_get_tree_record(&ctx, &got); + ASSERT_EQUAL(0x4400, got.root_addr); + ASSERT_EQUAL(9, got.first_unincorporated_generation); + ASSERT_EQUAL(TEST_LOG_L2.meta_addr, got.live_log.meta_addr); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(got.sealed_log)); + superblock_context_deinit(&ctx); +} + +/* + * A tree snapshot published while a sealed log is only partly incorporated must + * keep that sealed log: dropping it would strand the generations it still + * holds. + * + * This is the case a durability checkpoint hits when it commits a root that is + * newer than the sealed log's start but older than its end. L1 is sealed + * covering generations 0..5 (L2 starts at 6); a root whose first unincorporated + * generation is 5 has not yet folded in generation 5, so L1 must survive. Once + * a later root reaches 6, L1 is dropped. + */ +CTEST2(superblock, test_snapshot_preserves_unincorporated_sealed_log) +{ + superblock_context ctx; + platform_status rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_format(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + + // Get to a checkpoint mid-flight: install L1, then cut to L2, which moves L1 + // into the sealed slot. + superblock_log_cut(&ctx, TEST_LOG_L1); + superblock_log_cut(&ctx, TEST_LOG_L2); + rc = superblock_make_durable(&ctx); + ASSERT_TRUE(SUCCESS(rc)); + superblock_tree_record mid; + superblock_get_tree_record(&ctx, &mid); + ASSERT_EQUAL(TEST_LOG_L1.meta_addr, mid.sealed_log.meta_addr); + ASSERT_EQUAL(TEST_LOG_L2.meta_addr, mid.live_log.meta_addr); + + // Commit a root that stops short of L1's last generation: L1 must be kept. + superblock_snapshot_tree(&ctx, 0x4000, 5); + rc = superblock_make_durable(&ctx); + ASSERT_TRUE(SUCCESS(rc)); + + superblock_tree_record got; + superblock_get_tree_record(&ctx, &got); + ASSERT_EQUAL(0x4000, got.root_addr); + ASSERT_EQUAL(TEST_LOG_L2.meta_addr, got.live_log.meta_addr); + ASSERT_EQUAL(TEST_LOG_L1.meta_addr, got.sealed_log.meta_addr); + ASSERT_EQUAL(TEST_LOG_L1.start_generation, got.sealed_log.start_generation); + + // Now a root that covers all of L1's generations: it is dropped. + superblock_snapshot_tree(&ctx, 0x4400, 6); + rc = superblock_make_durable(&ctx); + ASSERT_TRUE(SUCCESS(rc)); + superblock_get_tree_record(&ctx, &got); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(got.sealed_log)); + + superblock_context_deinit(&ctx); +} + +/* + * A torn begin-checkpoint publish must leave the previous (steady) generation + * intact: mount falls back to {root R0, live L1, no sealed}, which recovery can + * replay -- never to a half-written checkpoint. + */ +CTEST2(superblock, test_two_log_checkpoint_torn_begin) +{ + superblock_context ctx; + platform_status rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_format(&ctx, &data->allocator_cfg); // slot0=gen1, slot1=gen2 + ASSERT_TRUE(SUCCESS(rc)); + + // Make the steady state durable into BOTH slots (gen3->slot0, gen4->slot1), + // so the fallback below is unambiguously the steady state, not the empty + // format one. + superblock_log_cut(&ctx, TEST_LOG_L1); + superblock_snapshot_tree(&ctx, 0x4000, 5); + rc = superblock_make_durable(&ctx); // gen3 -> slot0 + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_make_durable(&ctx); // gen4 -> slot1 + ASSERT_TRUE(SUCCESS(rc)); + + // Begin checkpoint: cut the log; this make_durable (gen5) targets slot0. + superblock_log_cut(&ctx, TEST_LOG_L2); + rc = superblock_make_durable(&ctx); // gen5 -> slot0 + ASSERT_TRUE(SUCCESS(rc)); + superblock_context_deinit(&ctx); + + // Tear the begin publish (newest slot, slot0/gen5). + superblock_test_corrupt_slot(data->ioh, data->io_cfg.page_size, 0); + + // Mount falls back to slot1 (gen4) = steady: live L1, no sealed. + rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_mount(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + superblock_tree_record got; + superblock_get_tree_record(&ctx, &got); + ASSERT_EQUAL(0x4000, got.root_addr); + ASSERT_EQUAL(TEST_LOG_L1.meta_addr, got.live_log.meta_addr); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(got.sealed_log)); + superblock_context_deinit(&ctx); +} + +/* + * The core torn-write guarantee: format writes gen 1 to slot 0 and gen 2 to + * slot 1, so the next publish targets slot 0 (gen 3) and carries the new tree + * record. Corrupting that newest slot must leave slot 1 (gen 2, which predates + * the record) intact, and mount must fall back to it. + */ +CTEST2(superblock, test_torn_write_falls_back_to_older_generation) +{ + superblock_context ctx; + platform_status rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_format(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + + // After format the image is gen 2 in slot 1, so this make_durable targets + // slot 0. Snapshot a nonempty root with no live log. + superblock_snapshot_tree(&ctx, 0x4000, 0); + rc = superblock_make_durable(&ctx); + ASSERT_TRUE(SUCCESS(rc)); + superblock_context_deinit(&ctx); + + // Simulate a torn write of the newest slot (slot 0, gen 3). + superblock_test_corrupt_slot(data->ioh, data->io_cfg.page_size, 0); + + // Mount must still succeed, falling back to slot 1 (gen 2), which carries + // the format's empty tree (root_addr 0), not the published root -- proving + // the older generation was left intact by the torn write. + rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_mount(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + + superblock_tree_record got; + superblock_get_tree_record(&ctx, &got); + ASSERT_EQUAL(0, got.root_addr); + superblock_context_deinit(&ctx); +} + +/* Both slots corrupt => no valid superblock. */ +CTEST2(superblock, test_both_slots_corrupt_is_not_found) +{ + superblock_context ctx; + platform_status rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_format(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + superblock_context_deinit(&ctx); + + superblock_test_corrupt_slot(data->ioh, data->io_cfg.page_size, 0); + superblock_test_corrupt_slot(data->ioh, data->io_cfg.page_size, 1); + + rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_mount(&ctx, &data->allocator_cfg); + ASSERT_FALSE(SUCCESS(rc)); + superblock_context_deinit(&ctx); +} + +/* + * The raw bootstrap geometry read (used before any subsystem is configured) + * returns the formatted geometry. + */ +CTEST2(superblock, test_read_geometry) +{ + superblock_context ctx; + platform_status rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_format(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + superblock_context_deinit(&ctx); + + disk_geometry geom; + rc = superblock_read_geometry(data->io_cfg.filename, &geom); + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_EQUAL(data->allocator_cfg.capacity, geom.disk_size); + ASSERT_EQUAL(data->io_cfg.page_size, geom.page_size); + ASSERT_EQUAL(data->io_cfg.extent_size, geom.extent_size); +}