From 2c21f8b27130920e09cbddc717cf06b5a43c1f87 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 17 Jul 2026 12:22:56 -0700 Subject: [PATCH 01/64] initial attempt Signed-off-by: Rob Johnson --- src/cache.h | 53 +- src/clockcache.c | 427 ++++++++++++++-- src/clockcache.h | 15 + src/core.c | 842 +++++++++++++++++++++++++------ src/core.h | 4 + src/log.h | 68 ++- src/memtable.c | 92 ++-- src/memtable.h | 29 ++ src/mini_allocator.c | 241 +++++++++ src/mini_allocator.h | 51 ++ src/platform_linux/laio.c | 30 ++ src/platform_linux/platform_io.h | 17 + src/rc_allocator.c | 574 +++++++++++++++++++-- src/rc_allocator.h | 61 ++- src/shard_log.c | 243 ++++++++- src/shard_log.h | 25 + src/splinterdb.c | 11 +- src/trunk.c | 47 +- src/trunk.h | 17 + tests/functional/btree_test.c | 199 ++++++++ tests/functional/cache_test.c | 698 +++++++++++++++++++++++++ tests/functional/log_test.c | 260 +++++++++- tests/unit/splinter_test.c | 79 +++ 23 files changed, 3779 insertions(+), 304 deletions(-) diff --git a/src/cache.h b/src/cache.h index bc239b7a..52bec143 100644 --- a/src/cache.h +++ b/src/cache.h @@ -97,6 +97,8 @@ cache_config_extent_page(const cache_config *cfg, uint64 extent_addr, uint64 i) typedef void (*cache_generic_fn)(cache *cc); typedef uint64 (*cache_generic_uint64_fn)(cache *cc); typedef void (*page_generic_fn)(cache *cc, page_handle *page); +typedef platform_status (*cache_durable_barrier_fn)(cache *cc); +typedef platform_status (*cache_writeback_fence_fn)(cache *cc); typedef page_handle *(*page_alloc_fn)(cache *cc, uint64 addr, page_type type); typedef void (*extent_discard_fn)(cache *cc, uint64 addr, page_type type); @@ -176,6 +178,8 @@ typedef struct cache_ops { page_sync_fn page_sync; extent_sync_fn extent_sync; cache_generic_fn flush; + cache_writeback_fence_fn writeback_fence; + cache_durable_barrier_fn durable_barrier; evict_fn evict; cache_generic_fn cleanup; page_addr_pred_fn in_use; @@ -366,8 +370,10 @@ cache_unclaim(cache *cc, page_handle *page) * * Blocks until outstanding read locks are released by other threads. * - * If you call this method, you almost certainly want to call - * cache_mark_dirty() immediately afterward. + * Clockcache conservatively begins a dirty interval on this transition, so a + * checkpoint fence cannot miss a caller that makes its first change before + * calling cache_mark_dirty(). cache_mark_dirty() remains the explicit, + * idempotent declaration of intent to modify the page. *---------------------------------------------------------------------- */ static inline void @@ -435,11 +441,10 @@ cache_prefetch_page(cache *cc, uint64 addr, page_type type) * The caller had better have the write lock on the page via cache_lock() * before changing its value. * - * TODO This method should be removed; its effect should come automatically - * with the acquisition of a write lock. @robj reports lots of bugs - * due to forgetting to call this method. And we can't think of a case - * where we'd want the "optimization" of taking a write lock but then - * decide not to dirty it. + * Clockcache already begins a dirty interval when the caller acquires the + * write lock; this call is therefore idempotent there. It remains part of the + * cache API to document the mutation and preserve the contract for other + * implementations. *---------------------------------------------------------------------- */ static inline void @@ -542,6 +547,40 @@ cache_flush(cache *cc) cc->ops->flush(cc); } +/* + *----------------------------------------------------------------------------- + * cache_writeback_fence + * + * Wait until every page whose current dirty interval began before this call's + * cut has completed writeback. Pages dirtied after the cut do not delay the + * call. The cache implementation must inspect resident metadata only; it must + * not fault pages in merely to satisfy the fence. + * + * This is the checkpointing primitive. It is intentionally narrower than + * cache_flush(), which attempts to leave the entire cache clean. + *----------------------------------------------------------------------------- + */ +static inline platform_status +cache_writeback_fence(cache *cc) +{ + return cc->ops->writeback_fence(cc); +} + +/* + *----------------------------------------------------------------------------- + * cache_durable_barrier + * + * Ensure that writeback completed before this call is durable across a power + * loss. Callers normally use this after cache_writeback_fence(), and again + * after publishing a checkpoint superblock. + *----------------------------------------------------------------------------- + */ +static inline platform_status +cache_durable_barrier(cache *cc) +{ + return cc->ops->durable_barrier(cc); +} + /* *----------------------------------------------------------------------------- * cache_evict diff --git a/src/clockcache.c b/src/clockcache.c index b8197236..74aba087 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -141,6 +141,9 @@ clockcache_print(platform_log_handle *log_handle, clockcache *cc); #define CC_LOADING (1u << 4) // page is actively being read from disk #define CC_WRITELOCKED (1u << 5) // write lock is held #define CC_CLAIMED (1u << 6) // claim is held +// A checkpoint fence has selected this dirty interval for writeback. New +// writers must not claim the page until that writeback completes. +#define CC_WRITEBACK_REQUESTED (1u << 7) /* Common status flag combinations */ // free entry @@ -222,6 +225,70 @@ clockcache_test_flag(clockcache *cc, uint32 entry_number, entry_status flag) return flag & clockcache_get_status(cc, entry_number); } +/* + *-------------------------------------------------------------------------- + * Dirty-generation bookkeeping + * + * A dirty generation identifies the clean->dirty interval, rather than every + * individual mutation. A checkpoint fence rotates the current generation and + * drains all older intervals. Writers cannot modify a page during writeback, + * so writing a later version of an older dirty interval still satisfies the + * fence. + *-------------------------------------------------------------------------- + */ +static void +clockcache_dirty_begin(clockcache *cc, uint32 entry_number) +{ + platform_status rc = platform_mutex_lock(&cc->dirty_lock); + platform_assert_status_ok(rc); + + if (clockcache_test_flag(cc, entry_number, CC_CLEAN)) { + clockcache_entry *entry = clockcache_get_entry(cc, entry_number); + debug_assert(entry->dirty_generation == 0); + entry->dirty_generation = + __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); + clockcache_clear_flag(cc, entry_number, CC_CLEAN); + } + + rc = platform_mutex_unlock(&cc->dirty_lock); + platform_assert_status_ok(rc); +} + +static void +clockcache_dirty_complete_writeback(clockcache *cc, uint32 entry_number) +{ + platform_status rc = platform_mutex_lock(&cc->dirty_lock); + platform_assert_status_ok(rc); + + clockcache_entry *entry = clockcache_get_entry(cc, entry_number); + debug_assert(entry->dirty_generation != 0); + entry->dirty_generation = 0; + debug_only uint32 was_clean = + clockcache_set_flag(cc, entry_number, CC_CLEAN); + debug_assert(!was_clean); + debug_only uint32 was_writeback = + clockcache_clear_flag(cc, entry_number, CC_WRITEBACK); + debug_assert(was_writeback); + clockcache_clear_flag(cc, entry_number, CC_WRITEBACK_REQUESTED); + + rc = platform_mutex_unlock(&cc->dirty_lock); + platform_assert_status_ok(rc); +} + +static void +clockcache_dirty_discard(clockcache *cc, uint32 entry_number) +{ + platform_status rc = platform_mutex_lock(&cc->dirty_lock); + platform_assert_status_ok(rc); + + clockcache_entry *entry = clockcache_get_entry(cc, entry_number); + entry->dirty_generation = 0; + clockcache_clear_flag(cc, entry_number, CC_WRITEBACK_REQUESTED); + + rc = platform_mutex_unlock(&cc->dirty_lock); + platform_assert_status_ok(rc); +} + #ifdef RECORD_ACQUISITION_STACKS static void clockcache_record_backtrace(clockcache *cc, uint32 entry_number) @@ -627,11 +694,27 @@ clockcache_try_get_claim(clockcache *cc, uint32 entry_number) entry_number, clockcache_test_flag(cc, entry_number, CC_CLAIMED)); + if (clockcache_test_flag(cc, entry_number, CC_WRITEBACK_REQUESTED)) { + return GET_RC_CONFLICT; + } + if (clockcache_set_flag(cc, entry_number, CC_CLAIMED)) { clockcache_log(0, entry_number, "return false\n", NULL); return GET_RC_CONFLICT; } + /* + * A fence may have selected the dirty interval between the first test and + * the claim. Do not let a new writer keep that selected interval dirty; + * drop the claim and let the requested writeback proceed instead. + */ + if (clockcache_test_flag(cc, entry_number, CC_WRITEBACK_REQUESTED)) { + debug_only uint32 was_claimed = + clockcache_clear_flag(cc, entry_number, CC_CLAIMED); + debug_assert(was_claimed); + return GET_RC_CONFLICT; + } + clockcache_record_backtrace(cc, entry_number); return GET_RC_SUCCESS; @@ -782,11 +865,16 @@ clockcache_try_get_write(clockcache *cc, uint32 entry_number) static inline bool32 clockcache_ok_to_writeback(clockcache *cc, uint32 entry_number, - bool32 with_access) + bool32 with_access, + bool32 requested_only) { uint32 status = clockcache_get_status(cc, entry_number); - return ((status == CC_CLEANABLE1_STATUS) - || (with_access && status == CC_CLEANABLE2_STATUS)); + uint32 request_flag = status & CC_WRITEBACK_REQUESTED; + uint32 base_status = status & ~CC_WRITEBACK_REQUESTED; + + return ((!requested_only || request_flag) + && ((base_status == CC_CLEANABLE1_STATUS) + || (with_access && base_status == CC_CLEANABLE2_STATUS))); } /* @@ -812,14 +900,24 @@ clockcache_try_set_writeback(clockcache *cc, volatile uint32 *status = &cc->entry[entry_number].status; if (__sync_bool_compare_and_swap( - status, CC_CLEANABLE1_STATUS, CC_WRITEBACK1_STATUS)) + status, CC_CLEANABLE1_STATUS, CC_WRITEBACK1_STATUS) + || __sync_bool_compare_and_swap(status, + CC_CLEANABLE1_STATUS + | CC_WRITEBACK_REQUESTED, + CC_WRITEBACK1_STATUS + | CC_WRITEBACK_REQUESTED)) { return TRUE; } if (with_access - && __sync_bool_compare_and_swap( - status, CC_CLEANABLE2_STATUS, CC_WRITEBACK2_STATUS)) + && (__sync_bool_compare_and_swap( + status, CC_CLEANABLE2_STATUS, CC_WRITEBACK2_STATUS) + || __sync_bool_compare_and_swap(status, + CC_CLEANABLE2_STATUS + | CC_WRITEBACK_REQUESTED, + CC_WRITEBACK2_STATUS + | CC_WRITEBACK_REQUESTED))) { return TRUE; } @@ -861,8 +959,6 @@ clockcache_write_callback(void *wbs) uint32 entry_number; clockcache_entry *entry; uint64 addr; - debug_only uint32 debug_status; - for (i = 0; i < count; i++) { entry_number = clockcache_data_to_entry_number(cc, (char *)iovec[i].iov_base); @@ -876,10 +972,7 @@ clockcache_write_callback(void *wbs) entry_number, addr); - debug_status = clockcache_set_flag(cc, entry_number, CC_CLEAN); - debug_assert(!debug_status); - debug_status = clockcache_clear_flag(cc, entry_number, CC_WRITEBACK); - debug_assert(debug_status); + clockcache_dirty_complete_writeback(cc, entry_number); } if (state->outstanding_pages) { @@ -922,8 +1015,11 @@ clockcache_abort_writeback_range(clockcache *cc, * they are not. *---------------------------------------------------------------------- */ -void -clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) +static platform_status +clockcache_batch_start_writeback(clockcache *cc, + uint64 batch, + bool32 is_urgent, + bool32 requested_only) { uint32 entry_no, next_entry_no; uint64 addr, first_addr, end_addr, i; @@ -932,6 +1028,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) uint64 end_entry_no = start_entry_no + CC_ENTRIES_PER_BATCH; clockcache_entry *entry, *next_entry; + platform_status result = STATUS_OK; debug_assert((tid < MAX_THREADS), "Invalid tid=%lu\n", tid); debug_assert(cc != NULL); @@ -953,7 +1050,8 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) entry = &cc->entry[entry_no]; addr = entry->page.disk_addr; // test and test and set in the if condition - if (clockcache_ok_to_writeback(cc, entry_no, is_urgent) + if (clockcache_ok_to_writeback( + cc, entry_no, is_urgent, requested_only) && clockcache_try_set_writeback(cc, entry_no, is_urgent)) { debug_assert(clockcache_lookup(cc, addr) == entry_no); @@ -968,6 +1066,8 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) next_entry_no = CC_UNMAPPED_ENTRY; } while ( next_entry_no != CC_UNMAPPED_ENTRY + && clockcache_ok_to_writeback( + cc, next_entry_no, is_urgent, requested_only) && clockcache_try_set_writeback(cc, next_entry_no, is_urgent)); first_addr += page_size; end_addr = entry->page.disk_addr; @@ -981,6 +1081,8 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) next_entry_no = CC_UNMAPPED_ENTRY; } while ( next_entry_no != CC_UNMAPPED_ENTRY + && clockcache_ok_to_writeback( + cc, next_entry_no, is_urgent, requested_only) && clockcache_try_set_writeback(cc, next_entry_no, is_urgent)); @@ -991,6 +1093,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) "clockcache_batch_start_writeback: async_io_state allocation " "failed\n"); clockcache_abort_writeback_range(cc, first_addr, end_addr); + result = STATUS_NO_MEMORY; goto close_log; } @@ -1008,6 +1111,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) platform_status_to_string(rc)); clockcache_abort_writeback_range(cc, first_addr, end_addr); platform_free(PROCESS_PRIVATE_HEAP_ID, state); + result = rc; goto close_log; } @@ -1033,6 +1137,7 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) io_async_state_deinit(state->iostate); clockcache_abort_writeback_range(cc, first_addr, end_addr); platform_free(PROCESS_PRIVATE_HEAP_ID, state); + result = rc; goto close_log; } } @@ -1042,12 +1147,156 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) cc->stats[tid].writes_issued++; } - io_async_run(state->iostate); + if (io_async_run(state->iostate) == ASYNC_STATUS_DONE) { + rc = io_async_state_get_result(state->iostate); + if (SUCCESS(rc)) { + platform_error_log( + "clockcache_batch_start_writeback: async write for addr " + "%lu completed without invoking its callback\n", + first_addr); + rc = STATUS_IO_ERROR; + } + clockcache_abort_writeback_range(cc, first_addr, end_addr); + io_async_state_deinit(state->iostate); + platform_free(PROCESS_PRIVATE_HEAP_ID, state); + result = rc; + goto close_log; + } } } close_log: clockcache_close_log_stream(); + return result; +} + +/* + *-------------------------------------------------------------------------- + * clockcache_fence_request_old_dirty -- + * + * Mark every dirty interval at or before cutoff for writeback. The dirty lock + * makes this atomic with both a clean->dirty transition and a writeback + * completion, so a false return means no interval from this fence remains. + *-------------------------------------------------------------------------- + */ +static bool32 +clockcache_fence_request_old_dirty(clockcache *cc, uint64 cutoff) +{ + bool32 pending = FALSE; + platform_status rc = platform_mutex_lock(&cc->dirty_lock); + platform_assert_status_ok(rc); + + for (uint32 entry_no = 0; entry_no < cc->cfg->page_capacity; entry_no++) { + clockcache_entry *entry = clockcache_get_entry(cc, entry_no); + if (entry->dirty_generation != 0 + && entry->dirty_generation <= cutoff) + { + clockcache_set_flag(cc, entry_no, CC_WRITEBACK_REQUESTED); + pending = TRUE; + } + } + + rc = platform_mutex_unlock(&cc->dirty_lock); + platform_assert_status_ok(rc); + return pending; +} + +/* + *-------------------------------------------------------------------------- + * clockcache_fence_cancel_requests -- + * + * Abandon a failed fence without leaving writers permanently blocked behind + * its request bits. A later fence will select these still-dirty generations + * again. + *-------------------------------------------------------------------------- + */ +static void +clockcache_fence_cancel_requests(clockcache *cc, uint64 cutoff) +{ + platform_status rc = platform_mutex_lock(&cc->dirty_lock); + platform_assert_status_ok(rc); + + for (uint32 entry_no = 0; entry_no < cc->cfg->page_capacity; entry_no++) { + clockcache_entry *entry = clockcache_get_entry(cc, entry_no); + if (entry->dirty_generation != 0 + && entry->dirty_generation <= cutoff) + { + clockcache_clear_flag(cc, entry_no, CC_WRITEBACK_REQUESTED); + } + } + + rc = platform_mutex_unlock(&cc->dirty_lock); + platform_assert_status_ok(rc); +} + +/* + *-------------------------------------------------------------------------- + * clockcache_fence_start_writeback_round -- + * + * Start writeback only for intervals selected by a checkpoint fence. This is + * a metadata scan of resident cache entries; no page lookup or trunk walk is + * involved. The normal clock hand may retain batch_busy while it owns a free + * batch, so a fence intentionally does not wait for that batch-level state. + * Per-entry compare-and-swap on CC_WRITEBACK provides the required exclusion + * from the normal cleaner. + *-------------------------------------------------------------------------- + */ +static platform_status +clockcache_fence_start_writeback_round(clockcache *cc) +{ + for (uint64 batch = 0; batch < cc->cfg->batch_capacity; batch++) { + platform_status rc = + clockcache_batch_start_writeback(cc, batch, TRUE, TRUE); + if (!SUCCESS(rc)) { + return rc; + } + } + return STATUS_OK; +} + +/* + *-------------------------------------------------------------------------- + * clockcache_writeback_fence -- + * + * Rotate the dirty generation, then wait only for dirty intervals that were + * already present at that cut. Pages dirtied after the cut carry the next + * generation and do not delay this checkpoint. A request bit blocks new + * writers from indefinitely extending a selected interval, while a writer + * that was already active is allowed to finish and its final bytes are what + * gets written. + *-------------------------------------------------------------------------- + */ +platform_status +clockcache_writeback_fence(clockcache *cc) +{ + platform_status rc = platform_mutex_lock(&cc->writeback_fence_lock); + platform_assert_status_ok(rc); + + rc = platform_mutex_lock(&cc->dirty_lock); + platform_assert_status_ok(rc); + uint64 cutoff = __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); + platform_assert(cutoff < UINT64_MAX); + __atomic_store_n(&cc->dirty_generation, cutoff + 1, __ATOMIC_RELEASE); + rc = platform_mutex_unlock(&cc->dirty_lock); + platform_assert_status_ok(rc); + + while (clockcache_fence_request_old_dirty(cc, cutoff)) { + rc = clockcache_fence_start_writeback_round(cc); + if (!SUCCESS(rc)) { + clockcache_fence_cancel_requests(cc, cutoff); + break; + } + /* + * Poll the local completion queue, then yield briefly if all selected + * pages are currently write-locked or in another cleaner's I/O. + */ + clockcache_wait(cc); + platform_sleep_ns(1000); + } + + platform_status unlock_rc = platform_mutex_unlock(&cc->writeback_fence_lock); + platform_assert_status_ok(unlock_rc); + return rc; } /* @@ -1148,6 +1397,7 @@ clockcache_try_evict(clockcache *cc, uint32 entry_number) /* 6. set status to CC_FREE_STATUS (clears claim and write lock) */ platform_assert(entry->waiters.head == NULL); entry->type = PAGE_TYPE_INVALID; + clockcache_dirty_discard(cc, entry_number); entry->status = CC_FREE_STATUS; clockcache_log( addr, entry_number, "evict: entry %u addr %lu\n", entry_number, addr); @@ -1228,7 +1478,12 @@ clockcache_move_hand(clockcache *cc, bool32 is_urgent) cleaner_hand = (evict_hand + cc->cleaner_gap) % cc->cfg->batch_capacity; clean_batch_busy = &cc->batch_busy[cleaner_hand]; if (__sync_bool_compare_and_swap(clean_batch_busy, FALSE, TRUE)) { - clockcache_batch_start_writeback(cc, cleaner_hand, is_urgent); + platform_status rc = clockcache_batch_start_writeback( + cc, cleaner_hand, is_urgent, FALSE); + if (!SUCCESS(rc)) { + platform_error_log("clockcache_move_hand: writeback failed: %s\n", + platform_status_to_string(rc)); + } was_busy = __sync_bool_compare_and_swap(clean_batch_busy, TRUE, FALSE); debug_assert(was_busy); } @@ -1251,7 +1506,8 @@ clockcache_get_free_page(clockcache *cc, uint32 status, page_type type, bool32 refcount, - bool32 blocking) + bool32 blocking, + bool32 begins_dirty) { uint32 entry_no; uint64 num_passes = 0; @@ -1261,6 +1517,7 @@ clockcache_get_free_page(clockcache *cc, timestamp wait_start; debug_assert((tid < MAX_THREADS), "Invalid tid=%lu\n", tid); + debug_assert(!begins_dirty || status == CC_ALLOC_STATUS); if (cc->per_thread[tid].free_hand == CC_UNMAPPED_ENTRY) { clockcache_move_hand(cc, FALSE); } @@ -1275,16 +1532,45 @@ clockcache_get_free_page(clockcache *cc, uint64 end_entry = start_entry + CC_ENTRIES_PER_BATCH; for (entry_no = start_entry; entry_no < end_entry; entry_no++) { entry = &cc->entry[entry_no]; - if (entry->status == CC_FREE_STATUS - && __sync_bool_compare_and_swap( - &entry->status, CC_FREE_STATUS, CC_ALLOC_STATUS)) - { + if (entry->status == CC_FREE_STATUS) { + platform_status rc = STATUS_OK; + if (begins_dirty) { + rc = platform_mutex_lock(&cc->dirty_lock); + platform_assert_status_ok(rc); + } + + bool32 reserved = __sync_bool_compare_and_swap( + &entry->status, CC_FREE_STATUS, CC_ALLOC_STATUS); + if (!reserved) { + if (begins_dirty) { + rc = platform_mutex_unlock(&cc->dirty_lock); + platform_assert_status_ok(rc); + } + continue; + } + + /* + * The allocation's dirty interval is tagged while holding the + * same lock as a fence cut. This prevents a checkpoint from + * seeing a dirty CC_ALLOC_STATUS entry with generation zero. + */ + debug_assert(entry->dirty_generation == 0); + if (begins_dirty) { + entry->dirty_generation = + __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); + } + entry->status = status; + entry->type = type; + + if (begins_dirty) { + rc = platform_mutex_unlock(&cc->dirty_lock); + platform_assert_status_ok(rc); + } + if (refcount) { clockcache_inc_ref(cc, entry_no, tid); } platform_assert(entry->waiters.head == NULL); - entry->status = status; - entry->type = type; debug_assert(entry->page.disk_addr == CC_UNMAPPED_ADDR); clockcache_record_backtrace(cc, entry_no); return entry_no; @@ -1343,7 +1629,9 @@ clockcache_flush(clockcache *cc) flush_hand < cc->cfg->page_capacity / CC_ENTRIES_PER_BATCH; flush_hand++) { - clockcache_batch_start_writeback(cc, flush_hand, TRUE); + platform_status rc = + clockcache_batch_start_writeback(cc, flush_hand, TRUE, FALSE); + platform_assert_status_ok(rc); } // make sure all aio is complete again @@ -1404,7 +1692,8 @@ clockcache_alloc(clockcache *cc, uint64 addr, page_type type) CC_ALLOC_STATUS, type, TRUE, // refcount - TRUE); // blocking + TRUE, // blocking + TRUE); // begins_dirty clockcache_entry *entry = &cc->entry[entry_no]; entry->page.disk_addr = addr; entry->type = type; @@ -1504,6 +1793,7 @@ clockcache_try_page_discard(clockcache *cc, uint64 addr) /* 6. set status to CC_FREE_STATUS (clears claim and write lock) */ platform_assert(entry->waiters.head == NULL); entry->type = PAGE_TYPE_INVALID; + clockcache_dirty_discard(cc, entry_number); entry->status = CC_FREE_STATUS; /* 7. reset pincount */ @@ -1638,7 +1928,8 @@ clockcache_acquire_entry_for_load(clockcache *cc, // IN CC_READ_LOADING_STATUS, type, TRUE, // refcount - TRUE); // blocking + TRUE, // blocking + FALSE); // begins_dirty clockcache_entry *entry = clockcache_get_entry(cc, entry_number); /* * If someone else is loading the page and has reserved the lookup, let them @@ -2166,6 +2457,7 @@ clockcache_lock(clockcache *cc, page_handle *page) entry_number, page->disk_addr); clockcache_get_write(cc, entry_number); + clockcache_dirty_begin(cc, entry_number); } void @@ -2202,7 +2494,7 @@ clockcache_mark_dirty(clockcache *cc, page_handle *page) "mark_dirty: entry %u addr %lu\n", entry_number, entry->page.disk_addr); - clockcache_clear_flag(cc, entry_number, CC_CLEAN); + clockcache_dirty_begin(cc, entry_number); return; } @@ -2266,9 +2558,26 @@ clockcache_page_sync(clockcache *cc, const threadid tid = platform_get_tid(); platform_status status; - if (!clockcache_try_set_writeback(cc, entry_number, TRUE)) { - platform_assert(clockcache_test_flag(cc, entry_number, CC_CLEAN)); - return; + while (!clockcache_try_set_writeback(cc, entry_number, TRUE)) { + if (clockcache_test_flag(cc, entry_number, CC_CLEAN)) { + return; + } + + /* + * A pressure cleaner or a checkpoint fence may have begun writeback + * after the caller released its claim. Wait for that writeback rather + * than treating a perfectly valid concurrent flush as an assertion. + */ + if (clockcache_test_flag(cc, entry_number, CC_WRITEBACK)) { + clockcache_wait(cc); + continue; + } + + platform_assert(0, + "page_sync requires a cleanable page: entry=%u " + "status=%u\n", + entry_number, + clockcache_get_status(cc, entry_number)); } if (cc->cfg->use_stats) { @@ -2330,11 +2639,7 @@ clockcache_page_sync(clockcache *cc, "page_sync write entry %u addr %lu\n", entry_number, addr); - debug_only uint8 rc; - rc = clockcache_set_flag(cc, entry_number, CC_CLEAN); - debug_assert(!rc); - rc = clockcache_clear_flag(cc, entry_number, CC_WRITEBACK); - debug_assert(rc); + clockcache_dirty_complete_writeback(cc, entry_number); } } @@ -2593,7 +2898,7 @@ clockcache_prefetch_pages(clockcache *cc, case GET_RC_EVICTED: { uint32 free_entry_no = clockcache_get_free_page( - cc, CC_READ_LOADING_STATUS, type, FALSE, TRUE); + cc, CC_READ_LOADING_STATUS, type, FALSE, TRUE, FALSE); clockcache_entry *entry = &cc->entry[free_entry_no]; entry->page.disk_addr = addr; entry->type = type; @@ -3171,6 +3476,20 @@ clockcache_flush_virtual(cache *c) clockcache_flush(cc); } +platform_status +clockcache_writeback_fence_virtual(cache *c) +{ + clockcache *cc = (clockcache *)c; + return clockcache_writeback_fence(cc); +} + +platform_status +clockcache_durable_barrier_virtual(cache *c) +{ + clockcache *cc = (clockcache *)c; + return io_durable_barrier(cc->io); +} + int clockcache_evict_all_virtual(cache *c, bool32 ignore_pinned) { @@ -3305,6 +3624,8 @@ static cache_ops clockcache_ops = { .page_sync = clockcache_page_sync_virtual, .extent_sync = clockcache_extent_sync_virtual, .flush = clockcache_flush_virtual, + .writeback_fence = clockcache_writeback_fence_virtual, + .durable_barrier = clockcache_durable_barrier_virtual, .evict = clockcache_evict_all_virtual, .cleanup = clockcache_wait_virtual, .in_use = clockcache_in_use_virtual, @@ -3397,9 +3718,28 @@ clockcache_init(clockcache *cc, // OUT cc->io = io; cc->heap_id = hid; + platform_status rc = platform_mutex_init(&cc->dirty_lock, mid, hid); + if (!SUCCESS(rc)) { + platform_error_log("clockcache_init: failed to initialize dirty lock: %s\n", + platform_status_to_string(rc)); + goto alloc_error; + } + + rc = platform_mutex_init(&cc->writeback_fence_lock, mid, hid); + if (!SUCCESS(rc)) { + platform_error_log( + "clockcache_init: failed to initialize writeback fence lock: %s\n", + platform_status_to_string(rc)); + platform_status destroy_rc = platform_mutex_destroy(&cc->dirty_lock); + platform_assert_status_ok(destroy_rc); + goto alloc_error; + } + cc->dirty_generation = 1; + cc->dirty_locks_initialized = TRUE; + /* lookup maps addrs to entries, entry contains the entries themselves */ - platform_status rc = platform_buffer_init( - &cc->lookup_bh, allocator_page_capacity * sizeof(cc->lookup[0])); + rc = platform_buffer_init(&cc->lookup_bh, + allocator_page_capacity * sizeof(cc->lookup[0])); if (!SUCCESS(rc)) { platform_error_log("clockcache_init: failed to allocate lookup table " "(%lu bytes): %s\n", @@ -3440,6 +3780,7 @@ clockcache_init(clockcache *cc, // OUT cc->data + clockcache_multiply_by_page_size(cc, i); cc->entry[i].page.disk_addr = CC_UNMAPPED_ADDR; cc->entry[i].status = CC_FREE_STATUS; + cc->entry[i].dirty_generation = 0; cc->entry[i].type = PAGE_TYPE_INVALID; async_wait_queue_init(&cc->entry[i].waiters); } @@ -3522,6 +3863,14 @@ clockcache_deinit(clockcache *cc) // IN/OUT #endif } + if (cc->dirty_locks_initialized) { + rc = platform_mutex_destroy(&cc->writeback_fence_lock); + platform_assert_status_ok(rc); + rc = platform_mutex_destroy(&cc->dirty_lock); + platform_assert_status_ok(rc); + cc->dirty_locks_initialized = FALSE; + } + if (cc->lookup) { rc = platform_buffer_deinit(&cc->lookup_bh); if (!SUCCESS(rc)) { diff --git a/src/clockcache.h b/src/clockcache.h index 814cde6a..1e601bd5 100644 --- a/src/clockcache.h +++ b/src/clockcache.h @@ -10,6 +10,7 @@ #pragma once #include "platform_buffer.h" +#include "platform_mutex.h" #include "platform_threads.h" #include "allocator.h" #include "cache.h" @@ -72,6 +73,9 @@ typedef uint32 entry_status; // Saved in clockcache_entry->status struct clockcache_entry { page_handle page; volatile entry_status status; + // Generation in which this page's current dirty interval began. Zero means + // no dirty interval (a clean page or an unpublished clean-load reservation). + volatile uint64 dirty_generation; page_type type; async_wait_queue waiters; #ifdef RECORD_ACQUISITION_STACKS @@ -139,6 +143,17 @@ struct clockcache { volatile bool32 *batch_busy; // Convenience pointer for batch_bh uint64 cleaner_gap; + /* + * Dirty-generation bookkeeping for checkpoint writeback fences. The dirty + * lock serializes the clean<->dirty transition with a fence cut and with + * writeback completion. Fence rounds are serialized separately so a page + * only needs one outstanding writeback-request bit. + */ + platform_mutex dirty_lock; + platform_mutex writeback_fence_lock; + uint64 dirty_generation; + bool32 dirty_locks_initialized; + volatile struct { volatile uint32 free_hand; bool32 enable_sync_get; diff --git a/src/core.c b/src/core.c index d499f24a..3faac28e 100644 --- a/src/core.c +++ b/src/core.c @@ -46,8 +46,38 @@ static const int64 latency_histo_buckets[LATENCYHISTO_SIZE] = { _Static_assert(CORE_NUM_MEMTABLES <= MAX_MEMTABLES, "CORE_NUM_MEMTABLES <= MAX_MEMTABLES"); -/* Some randomly chosen Splinter super-block checksum seed. */ -#define CORE_SUPER_CSUM_SEED (42) +/* Checkpoint metadata has independently checksummed directory and records. */ +#define CORE_CHECKPOINT_DIRECTORY_CSUM_SEED (42) +#define CORE_CHECKPOINT_RECORD_CSUM_SEED (43) + +#define CORE_CHECKPOINT_FORMAT_VERSION (2) +#define CORE_CHECKPOINT_RECORD_COUNT (2) + +#define CORE_CHECKPOINT_DIRECTORY_MAGIC (0x534442434B505444ULL) // SDBCKPTD +#define CORE_CHECKPOINT_RECORD_MAGIC (0x534442434B505452ULL) // SDBCKPTR + +static platform_status +core_checkpoint_lock_init(core_handle *spl) +{ + platform_status rc = platform_mutex_init(&spl->checkpoint_lock, + platform_get_module_id(), + spl->heap_id); + if (SUCCESS(rc)) { + spl->checkpoint_lock_initialized = TRUE; + } + return rc; +} + +static void +core_checkpoint_lock_deinit(core_handle *spl) +{ + if (!spl->checkpoint_lock_initialized) { + return; + } + platform_status rc = platform_mutex_destroy(&spl->checkpoint_lock); + platform_assert_status_ok(rc); + spl->checkpoint_lock_initialized = FALSE; +} /* * core logging functions. @@ -107,142 +137,542 @@ core_close_log_stream_if_enabled(core_handle *spl, /* *----------------------------------------------------------------------------- - * Splinter Super Block: Disk-resident structure. - * Super block lives on page of page type == PAGE_TYPE_SUPERBLOCK. + * Checkpoint metadata: disk-resident structures. + * + * allocator_get_super_addr() identifies a fixed page in the allocator's + * bootstrap extent. That page is an immutable directory, written exactly + * once when the table is created. The directory names two independently + * allocated record extents. Checkpoint publication alternates between their + * first pages, leaving one formerly valid record untouched if a new write is + * torn. + * + * This is intentionally a new on-disk format. Do not interpret a legacy + * core_super_block as a directory: doing so would turn arbitrary old fields + * into allocator-owned addresses. Existing databases must be migrated or + * reformatted before using this checkpoint metadata format. *----------------------------------------------------------------------------- */ -typedef struct ONDISK core_super_block { - uint64 root_addr; // Address of the root of the trunk for the instance - // referenced by this superblock. - uint64 log_addr; - uint64 log_meta_addr; - uint64 timestamp; - bool32 checkpointed; - bool32 unmounted; +typedef struct ONDISK core_checkpoint_directory { + uint64 magic; + uint64 format_version; + uint64 table_id; + uint64 record_addr[CORE_CHECKPOINT_RECORD_COUNT]; checksum128 checksum; -} core_super_block; +} core_checkpoint_directory; -/* - *----------------------------------------------------------------------------- - * Super block functions - *----------------------------------------------------------------------------- - */ -static platform_status -core_set_super_block(core_handle *spl, - bool32 is_checkpoint, - bool32 is_unmount, - bool32 is_create) +typedef struct ONDISK core_checkpoint_record { + /* + * The highest memtable generation incorporated in root_addr. The boolean + * keeps the fresh-database case distinct from generation zero. + */ + uint64 incorporated_generation; + bool32 has_incorporated_generation; + uint64 root_addr; + uint64 timestamp; + uint64 sequence; + uint64 table_id; + uint32 record_slot; + bool32 checkpointed; + bool32 unmounted; + uint64 magic; + uint64 format_version; + checksum128 checksum; +} core_checkpoint_record; + +typedef struct core_checkpoint_records { + core_checkpoint_record record[CORE_CHECKPOINT_RECORD_COUNT]; + bool32 valid[CORE_CHECKPOINT_RECORD_COUNT]; + bool32 have_newest; + uint64 newest_slot; + bool32 have_newest_unmounted; + uint64 newest_unmounted_slot; +} core_checkpoint_records; + +static checksum128 +core_checkpoint_directory_checksum(const core_checkpoint_directory *directory) { - uint64 super_addr; - page_handle *super_page; - core_super_block *super; - uint64 wait = 1; - platform_status rc; + return platform_checksum128(directory, + offsetof(core_checkpoint_directory, checksum), + CORE_CHECKPOINT_DIRECTORY_CSUM_SEED); +} - if (is_create) { - rc = allocator_alloc_super_addr(spl->al, spl->id, &super_addr); - } else { - rc = allocator_get_super_addr(spl->al, spl->id, &super_addr); +static checksum128 +core_checkpoint_record_checksum(const core_checkpoint_record *record) +{ + return platform_checksum128(record, + offsetof(core_checkpoint_record, checksum), + CORE_CHECKPOINT_RECORD_CSUM_SEED); +} + +static bool32 +core_checkpoint_record_addr_is_valid(core_handle *spl, uint64 addr) +{ + allocator_config *allocator_cfg = allocator_get_config(spl->al); + uint64 page_size = cache_page_size(spl->cc); + + return addr != 0 && addr % allocator_cfg->io_cfg->extent_size == 0 + && addr < allocator_cfg->capacity + && page_size <= allocator_cfg->capacity - addr; +} + +static bool32 +core_checkpoint_directory_is_valid(core_handle *spl, + const core_checkpoint_directory *directory) +{ + if (directory->magic != CORE_CHECKPOINT_DIRECTORY_MAGIC + || directory->format_version != CORE_CHECKPOINT_FORMAT_VERSION + || directory->table_id != spl->id + || !platform_checksum_is_equal( + directory->checksum, core_checkpoint_directory_checksum(directory))) + { + return FALSE; } + + uint64 record0 = directory->record_addr[0]; + uint64 record1 = directory->record_addr[1]; + allocator_config *allocator_cfg = allocator_get_config(spl->al); + return core_checkpoint_record_addr_is_valid(spl, record0) + && core_checkpoint_record_addr_is_valid(spl, record1) + && record0 != record1 + && !allocator_config_pages_share_extent(allocator_cfg, record0, record1); +} + +static bool32 +core_checkpoint_record_is_valid(core_handle *spl, + const core_checkpoint_record *record, + uint64 record_slot) +{ + return record->magic == CORE_CHECKPOINT_RECORD_MAGIC + && record->format_version == CORE_CHECKPOINT_FORMAT_VERSION + && record->table_id == spl->id && record->record_slot == record_slot + && record->sequence != 0 + && (record->has_incorporated_generation == FALSE + || record->has_incorporated_generation == TRUE) + && (record->has_incorporated_generation + ? record->incorporated_generation < UINT64_MAX + : record->incorporated_generation == 0) + && platform_checksum_is_equal( + record->checksum, core_checkpoint_record_checksum(record)); +} + +static void +core_write_checkpoint_page(core_handle *spl, + uint64 page_addr, + const void *contents, + uint64 contents_size) +{ + page_handle *page = + cache_get(spl->cc, page_addr, TRUE, PAGE_TYPE_SUPERBLOCK); + uint64 wait = 1; + while (!cache_try_claim(spl->cc, page)) { + cache_unget(spl->cc, page); + platform_sleep_ns(wait); + wait = wait > 1024 ? wait : 2 * wait; + page = cache_get(spl->cc, page_addr, TRUE, PAGE_TYPE_SUPERBLOCK); + } + cache_lock(spl->cc, page); + platform_assert(contents_size <= cache_page_size(spl->cc)); + memset(page->data, 0, cache_page_size(spl->cc)); + memcpy(page->data, contents, contents_size); + cache_mark_dirty(spl->cc, page); + cache_unlock(spl->cc, page); + cache_unclaim(spl->cc, page); + cache_page_sync(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); + cache_unget(spl->cc, page); +} + +static void +core_initialize_checkpoint_record_page(core_handle *spl, uint64 page_addr) +{ + page_handle *page = cache_alloc(spl->cc, page_addr, PAGE_TYPE_SUPERBLOCK); + platform_assert(page != NULL); + memset(page->data, 0, cache_page_size(spl->cc)); + cache_mark_dirty(spl->cc, page); + cache_unlock(spl->cc, page); + cache_unclaim(spl->cc, page); + cache_page_sync(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); + cache_unget(spl->cc, page); +} + +static platform_status +core_create_checkpoint_directory(core_handle *spl, + core_checkpoint_directory *directory) +{ + uint64 directory_addr; + platform_status rc = + allocator_alloc_super_addr(spl->al, spl->id, &directory_addr); if (!SUCCESS(rc)) { - platform_error_log("core_set_super_block: failed to %s super block " - "address for root id %lu: %s\n", - is_create ? "allocate" : "get", + platform_error_log("core_create_checkpoint_directory: failed to allocate " + "directory address for root id %lu: %s\n", spl->id, platform_status_to_string(rc)); return rc; } - super_page = cache_get(spl->cc, super_addr, TRUE, PAGE_TYPE_SUPERBLOCK); - while (!cache_try_claim(spl->cc, super_page)) { - platform_sleep_ns(wait); - wait *= 2; + + ZERO_CONTENTS(directory); + directory->magic = CORE_CHECKPOINT_DIRECTORY_MAGIC; + directory->format_version = CORE_CHECKPOINT_FORMAT_VERSION; + directory->table_id = spl->id; + + for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { + uint64 record_addr; + rc = allocator_alloc(spl->al, &record_addr, PAGE_TYPE_SUPERBLOCK); + if (!SUCCESS(rc)) { + platform_error_log("core_create_checkpoint_directory: failed to " + "allocate record extent %lu: %s\n", + slot, + platform_status_to_string(rc)); + return rc; + } + directory->record_addr[slot] = record_addr; + core_initialize_checkpoint_record_page(spl, record_addr); } - wait = 1; - cache_lock(spl->cc, super_page); - super = (core_super_block *)super_page->data; - uint64 old_root_addr = super->root_addr; + directory->checksum = core_checkpoint_directory_checksum(directory); + core_write_checkpoint_page( + spl, directory_addr, directory, sizeof(*directory)); - trunk_ondisk_node_handle root_handle; - trunk_init_root_handle(&spl->trunk_context, &root_handle); - uint64 root_addr = trunk_ondisk_node_handle_addr(&root_handle); - if (root_addr != 0) { - trunk_inc_ref(spl->al, root_addr); + /* + * Submit the newly initialized record and directory pages before the + * durable barrier. allocator_alloc() changes its refcount map only in + * memory; crash recovery deliberately rebuilds that map instead of relying + * on this publication. allocator_alloc_super_addr() does write the raw + * bootstrap mapping through the shared backing I/O handle, which the + * following durable barrier fdatasyncs with the directory page. + */ + rc = cache_writeback_fence(spl->cc); + if (!SUCCESS(rc)) { + return rc; } - super->root_addr = root_addr; - trunk_ondisk_node_handle_deinit(&root_handle); + return cache_durable_barrier(spl->cc); +} - if (spl->cfg.use_log) { - if (spl->log) { - super->log_addr = log_addr(spl->log); - super->log_meta_addr = log_meta_addr(spl->log); - } else { - super->log_addr = 0; - super->log_meta_addr = 0; +static platform_status +core_get_checkpoint_directory(core_handle *spl, + core_checkpoint_directory *directory) +{ + uint64 directory_addr; + platform_status rc = + allocator_get_super_addr(spl->al, spl->id, &directory_addr); + if (!SUCCESS(rc)) { + return rc; + } + + page_handle *page = + cache_get(spl->cc, directory_addr, TRUE, PAGE_TYPE_SUPERBLOCK); + memcpy(directory, page->data, sizeof(*directory)); + cache_unget(spl->cc, page); + + if (!core_checkpoint_directory_is_valid(spl, directory)) { + platform_error_log("core_get_checkpoint_directory: no compatible " + "checkpoint directory for root id %lu\n", + spl->id); + return STATUS_BAD_PARAM; + } + return STATUS_OK; +} + +static platform_status +core_load_checkpoint_records(core_handle *spl, + const core_checkpoint_directory *directory, + core_checkpoint_records *records) +{ + ZERO_CONTENTS(records); + for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { + page_handle *page = cache_get(spl->cc, + directory->record_addr[slot], + TRUE, + PAGE_TYPE_SUPERBLOCK); + memcpy(&records->record[slot], page->data, sizeof(records->record[slot])); + cache_unget(spl->cc, page); + + records->valid[slot] = core_checkpoint_record_is_valid( + spl, &records->record[slot], slot); + if (!records->valid[slot]) { + continue; + } + + if (!records->have_newest + || records->record[records->newest_slot].sequence + < records->record[slot].sequence) + { + records->have_newest = TRUE; + records->newest_slot = slot; + } + if (records->record[slot].unmounted + && (!records->have_newest_unmounted + || records->record[records->newest_unmounted_slot].sequence + < records->record[slot].sequence)) + { + records->have_newest_unmounted = TRUE; + records->newest_unmounted_slot = slot; } } - super->timestamp = platform_get_real_time(); - super->checkpointed = is_checkpoint; - super->unmounted = is_unmount; - super->checksum = - platform_checksum128(super, - sizeof(core_super_block) - sizeof(checksum128), - CORE_SUPER_CSUM_SEED); - - cache_mark_dirty(spl->cc, super_page); - cache_unlock(spl->cc, super_page); - cache_unclaim(spl->cc, super_page); - cache_unget(spl->cc, super_page); - cache_page_sync(spl->cc, super_page, TRUE, PAGE_TYPE_SUPERBLOCK); - - if (old_root_addr != 0 && !is_create) { + + if (records->valid[0] && records->valid[1] + && records->record[0].sequence == records->record[1].sequence) + { + platform_error_log("core_load_checkpoint_records: duplicate record " + "sequence %lu for root id %lu\n", + records->record[0].sequence, + spl->id); + return STATUS_BAD_PARAM; + } + return STATUS_OK; +} + +static void +core_destroy_checkpoint_record_extent(core_handle *spl, uint64 record_addr) +{ + refcount ref = + allocator_dec_ref(spl->al, record_addr, PAGE_TYPE_SUPERBLOCK); + if (ref != AL_NO_REFS) { + platform_error_log("core_destroy_checkpoint_record_extent: record extent " + "%lu has unexpected refcount %u\n", + record_addr, + ref); + return; + } + + cache_extent_discard(spl->cc, record_addr, PAGE_TYPE_SUPERBLOCK); + ref = allocator_dec_ref(spl->al, record_addr, PAGE_TYPE_SUPERBLOCK); + platform_assert(ref == AL_FREE); +} + +static void +core_destroy_checkpoint_storage(core_handle *spl) +{ + core_checkpoint_directory directory; + platform_status rc = core_get_checkpoint_directory(spl, &directory); + if (!SUCCESS(rc)) { + platform_error_log("core_destroy_checkpoint_storage: unable to load " + "checkpoint directory for root id %lu: %s\n", + spl->id, + platform_status_to_string(rc)); + return; + } + + core_checkpoint_records records; + rc = core_load_checkpoint_records(spl, &directory, &records); + if (!SUCCESS(rc)) { + platform_error_log("core_destroy_checkpoint_storage: unable to load " + "checkpoint records for root id %lu: %s\n", + spl->id, + platform_status_to_string(rc)); + return; + } + + /* + * Both valid slots own independent root references. Keeping the older + * record live makes it a real fallback if the next record write is torn; + * its reference is released only when that slot is successfully + * overwritten. This is clean destruction, so release both record owners. + */ + for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { + if (!records.valid[slot] || records.record[slot].root_addr == 0) { + continue; + } rc = trunk_dec_ref(spl->cfg.trunk_node_cfg, PROCESS_PRIVATE_HEAP_ID, spl->cc, spl->al, spl->ts, - old_root_addr); + records.record[slot].root_addr); if (!SUCCESS(rc)) { - platform_error_log("core_set_super_block: trunk_dec_ref failed for " - "old root addr %lu: %s\n", - old_root_addr, + platform_error_log("core_destroy_checkpoint_storage: failed to " + "release record %lu root %lu: %s\n", + slot, + records.record[slot].root_addr, platform_status_to_string(rc)); - return rc; } } + for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { + core_destroy_checkpoint_record_extent(spl, directory.record_addr[slot]); + } +} + +/* + *----------------------------------------------------------------------------- + * Checkpoint record functions + *----------------------------------------------------------------------------- + */ +static platform_status +core_capture_checkpoint_cut(core_handle *spl, + trunk_snapshot *snapshot, + bool32 *has_incorporated_generation, + uint64 *incorporated_generation) +{ + /* + * Incorporation publishes its generation and root while holding lookup + * exclusion before taking the trunk root lock. Take the checkpoint cut + * in the same order, so a record never combines a pre-incorporation + * generation with a post-incorporation root (or the converse). + */ + memtable_block_lookups(&spl->mt_ctxt); + uint64 retired_generation = memtable_generation_retired(&spl->mt_ctxt); + platform_status rc = trunk_snapshot_acquire(&spl->trunk_context, snapshot); + memtable_unblock_lookups(&spl->mt_ctxt); + if (!SUCCESS(rc)) { + return rc; + } + + *has_incorporated_generation = retired_generation != UINT64_MAX; + *incorporated_generation = *has_incorporated_generation + ? retired_generation + : 0; return STATUS_OK; } -static core_super_block * -core_get_super_block_if_valid(core_handle *spl, page_handle **super_page) +static platform_status +core_publish_checkpoint_record(core_handle *spl, + bool32 is_checkpoint, + bool32 is_unmount, + bool32 is_create) { - uint64 super_addr; - core_super_block *super; + uint64 old_root_addr; + platform_status rc; + trunk_snapshot snapshot; + bool32 has_incorporated_generation; + uint64 incorporated_generation; + core_checkpoint_directory directory; + core_checkpoint_records records; + uint64 target_slot; + core_checkpoint_record record; - platform_status rc = allocator_get_super_addr(spl->al, spl->id, &super_addr); - platform_assert_status_ok(rc); - *super_page = cache_get(spl->cc, super_addr, TRUE, PAGE_TYPE_SUPERBLOCK); - super = (core_super_block *)(*super_page)->data; - - if (!platform_checksum_is_equal( - super->checksum, - platform_checksum128(super, - sizeof(core_super_block) - sizeof(checksum128), - CORE_SUPER_CSUM_SEED))) + /* + * The snapshot, target-slot selection, durable record write, and old-slot + * release are one publication transaction. In particular, two concurrent + * publishers must never choose the same target slot or release the same + * former record owner. + */ + rc = platform_mutex_lock(&spl->checkpoint_lock); + if (!SUCCESS(rc)) { + return rc; + } + + rc = core_capture_checkpoint_cut(spl, + &snapshot, + &has_incorporated_generation, + &incorporated_generation); + if (!SUCCESS(rc)) { + goto unlock_checkpoint; + } + + /* + * The snapshot reference makes the root stable, but not necessarily + * durable. Drain only the cache intervals that existed at this cut before + * making a record that can name the root durable. This can incidentally + * persist newer log/data pages, but it does not seal or publish a logical + * durable-log tail; tail sync is a separate operation. + */ + rc = trunk_make_durable(&spl->trunk_context); + if (!SUCCESS(rc)) { + goto release_snapshot; + } + + if (is_create) { + rc = core_create_checkpoint_directory(spl, &directory); + } else { + rc = core_get_checkpoint_directory(spl, &directory); + } + if (!SUCCESS(rc)) { + platform_error_log("core_publish_checkpoint_record: failed to %s " + "checkpoint directory for root id %lu: %s\n", + is_create ? "create" : "load", + spl->id, + platform_status_to_string(rc)); + goto release_snapshot; + } + + rc = core_load_checkpoint_records(spl, &directory, &records); + if (!SUCCESS(rc)) { + goto release_snapshot; + } + + if (records.have_newest + && records.record[records.newest_slot].sequence == UINT64_MAX) { - cache_unget(spl->cc, *super_page); - *super_page = NULL; - return NULL; + rc = STATUS_LIMIT_EXCEEDED; + goto release_snapshot; + } + target_slot = records.have_newest ? records.newest_slot ^ 1 : 0; + /* + * Each valid record owns its root reference. This publication overwrites + * target_slot, so retain that slot's old root until the replacement page + * is durable, then release only the overwritten owner. The newest record + * remains independently live as the torn-write fallback. + */ + old_root_addr = records.valid[target_slot] + ? records.record[target_slot].root_addr + : 0; + + ZERO_CONTENTS(&record); + record.incorporated_generation = incorporated_generation; + record.has_incorporated_generation = has_incorporated_generation; + record.root_addr = snapshot.root_addr; + record.timestamp = platform_get_real_time(); + record.sequence = records.have_newest + ? records.record[records.newest_slot].sequence + 1 + : 1; + record.table_id = spl->id; + record.record_slot = target_slot; + record.checkpointed = is_checkpoint; + record.unmounted = is_unmount; + record.magic = CORE_CHECKPOINT_RECORD_MAGIC; + record.format_version = CORE_CHECKPOINT_FORMAT_VERSION; + + record.checksum = core_checkpoint_record_checksum(&record); + + core_write_checkpoint_page(spl, + directory.record_addr[target_slot], + &record, + sizeof(record)); + /* The record now owns this reference, even if the barrier reports failure. */ + snapshot.root_addr = 0; + + rc = cache_durable_barrier(spl->cc); + if (!SUCCESS(rc)) { + /* The new record may be durable, so retain its transferred root ref. */ + goto unlock_checkpoint; } - return super; -} + if (old_root_addr != 0) { + rc = trunk_dec_ref(spl->cfg.trunk_node_cfg, + PROCESS_PRIVATE_HEAP_ID, + spl->cc, + spl->al, + spl->ts, + old_root_addr); + if (!SUCCESS(rc)) { + platform_error_log("core_publish_checkpoint_record: trunk_dec_ref " + "failed for old root addr %lu: %s\n", + old_root_addr, + platform_status_to_string(rc)); + goto unlock_checkpoint; + } + } -static void -core_release_super_block(core_handle *spl, page_handle *super_page) -{ - cache_unget(spl->cc, super_page); + rc = STATUS_OK; + goto unlock_checkpoint; + +release_snapshot: + { + platform_status release_rc = + trunk_snapshot_release(&spl->trunk_context, &snapshot); + if (SUCCESS(rc) && !SUCCESS(release_rc)) { + rc = release_rc; + } + } + +unlock_checkpoint: + { + platform_status unlock_rc = platform_mutex_unlock(&spl->checkpoint_lock); + if (SUCCESS(rc) && !SUCCESS(unlock_rc)) { + rc = unlock_rc; + } + } + return rc; } /* @@ -418,6 +848,7 @@ core_begin_memtable_insert(core_handle *spl, uint64 *generation, memtable **mt) static platform_status core_log_insert(core_handle *spl, + uint64 memtable_generation, key tuple_key, message msg, const btree_insert_results *insert_results) @@ -436,8 +867,11 @@ core_log_insert(core_handle *spl, merge_accumulator_is_null(&insert_results->msg_blob) ? msg : merge_accumulator_to_message(&insert_results->msg_blob); - int log_rc = - log_write(spl->log, tuple_key, log_msg, insert_results->leaf_generation); + int log_rc = log_write(spl->log, + tuple_key, + log_msg, + memtable_generation, + insert_results->leaf_generation); return log_rc == 0 ? STATUS_OK : (platform_status){.r = log_rc}; } @@ -1554,7 +1988,7 @@ core_insert(core_handle *spl, goto end_insert; } - rc = core_log_insert(spl, tuple_key, data, &insert_results); + rc = core_log_insert(spl, generation, tuple_key, data, &insert_results); if (!SUCCESS(rc)) { goto end_insert; } @@ -1879,18 +2313,25 @@ core_mkfs(core_handle *spl, spl->heap_id = hid; spl->ts = ts; + platform_status rc = core_checkpoint_lock_init(spl); + if (!SUCCESS(rc)) { + platform_error_log("core_mkfs: checkpoint lock initialization failed: %s\n", + platform_status_to_string(rc)); + return rc; + } + // set up the memtable context memtable_config *mt_cfg = &spl->cfg.mt_cfg; - platform_status rc = memtable_context_init(&spl->mt_ctxt, - spl->heap_id, - cc, - mt_cfg, - core_memtable_flush_virtual, - spl); + rc = memtable_context_init(&spl->mt_ctxt, + spl->heap_id, + cc, + mt_cfg, + core_memtable_flush_virtual, + spl); if (!SUCCESS(rc)) { platform_error_log("core_mkfs: memtable_context_init failed: %s\n", platform_status_to_string(rc)); - return rc; + goto deinit_checkpoint_lock; } // set up the log @@ -1918,9 +2359,9 @@ core_mkfs(core_handle *spl, goto deinit_trunk_context; } - rc = core_set_super_block(spl, FALSE, FALSE, TRUE); + rc = core_publish_checkpoint_record(spl, FALSE, FALSE, TRUE); if (!SUCCESS(rc)) { - platform_error_log("core_mkfs: core_set_super_block failed: %s\n", + platform_error_log("core_mkfs: core_publish_checkpoint_record failed: %s\n", platform_status_to_string(rc)); goto deinit_stats; } @@ -1937,6 +2378,8 @@ core_mkfs(core_handle *spl, } deinit_memtable_context: memtable_context_deinit(&spl->mt_ctxt); +deinit_checkpoint_lock: + core_checkpoint_lock_deinit(spl); return rc; } @@ -1962,30 +2405,71 @@ core_mount(core_handle *spl, spl->heap_id = hid; spl->ts = ts; - // find the unmounted super block - uint64 root_addr = 0; - uint64 latest_timestamp = 0; - page_handle *super_page; - core_super_block *super = core_get_super_block_if_valid(spl, &super_page); - if (super != NULL) { - if (super->unmounted && super->timestamp > latest_timestamp) { - root_addr = super->root_addr; - latest_timestamp = super->timestamp; - } - core_release_super_block(spl, super_page); + platform_status rc = core_checkpoint_lock_init(spl); + if (!SUCCESS(rc)) { + platform_error_log("core_mount: checkpoint lock initialization failed: %s\n", + platform_status_to_string(rc)); + return rc; } + /* + * Preserve the historical clean-only mount rule for this first format + * slice: an interrupted run is not replayed yet, so only an explicitly + * unmounted record supplies the root. We still validate both records and + * choose the newest clean one by sequence rather than wall-clock time. + */ + uint64 root_addr = 0; + bool32 has_incorporated_generation = FALSE; + uint64 incorporated_generation = 0; + core_checkpoint_directory directory; + core_checkpoint_records records; + rc = core_get_checkpoint_directory(spl, &directory); + if (!SUCCESS(rc)) { + goto deinit_checkpoint_lock; + } + rc = core_load_checkpoint_records(spl, &directory, &records); + if (!SUCCESS(rc)) { + goto deinit_checkpoint_lock; + } + if (!records.have_newest) { + platform_error_log("core_mount: checkpoint directory for root id %lu " + "has no valid records\n", + spl->id); + rc = STATUS_BAD_PARAM; + goto deinit_checkpoint_lock; + } + const core_checkpoint_record *record = + &records.record[records.newest_slot]; + if (!record->unmounted) { + /* + * This is an interrupted run. An older clean record is only an A/B + * torn-write fallback, not permission to silently discard the newer + * checkpoint and its log suffix. Do not overwrite its metadata before + * log replay and allocator reconstruction are wired. + */ + platform_error_log("core_mount: root id %lu requires crash recovery\n", + spl->id); + rc = STATUS_INVALID_STATE; + goto deinit_checkpoint_lock; + } + root_addr = record->root_addr; + has_incorporated_generation = record->has_incorporated_generation; + incorporated_generation = record->incorporated_generation; + memtable_config *mt_cfg = &spl->cfg.mt_cfg; - platform_status rc = memtable_context_init(&spl->mt_ctxt, - spl->heap_id, - cc, - mt_cfg, - core_memtable_flush_virtual, - spl); + rc = memtable_context_init_at_generation( + &spl->mt_ctxt, + spl->heap_id, + cc, + mt_cfg, + core_memtable_flush_virtual, + spl, + has_incorporated_generation ? incorporated_generation + 1 : 0); if (!SUCCESS(rc)) { - platform_error_log("core_mount: memtable_context_init failed: %s\n", + platform_error_log("core_mount: memtable_context_init_at_generation " + "failed: %s\n", platform_status_to_string(rc)); - return rc; + goto deinit_checkpoint_lock; } if (spl->cfg.use_log) { @@ -2012,9 +2496,9 @@ core_mount(core_handle *spl, goto deinit_trunk_context; } - rc = core_set_super_block(spl, FALSE, FALSE, FALSE); + rc = core_publish_checkpoint_record(spl, FALSE, FALSE, FALSE); if (!SUCCESS(rc)) { - platform_error_log("core_mount: core_set_super_block failed: %s\n", + platform_error_log("core_mount: core_publish_checkpoint_record failed: %s\n", platform_status_to_string(rc)); goto deinit_stats; } @@ -2031,6 +2515,8 @@ core_mount(core_handle *spl, } deinit_memtable_context: memtable_context_deinit(&spl->mt_ctxt); +deinit_checkpoint_lock: + core_checkpoint_lock_deinit(spl); return rc; } @@ -2082,9 +2568,11 @@ core_report_unincorporated_memtables(core_handle *spl) /* * This function is only safe to call when all other calls to spl have returned. + * It intentionally leaves the memtable and log contexts live: the clean + * checkpoint record needs both after final incorporation has quiesced. */ -void -core_prepare_for_shutdown(core_handle *spl) +static void +core_quiesce_for_shutdown(core_handle *spl) { // write current memtable to disk // (any others must already be flushing/flushed) @@ -2104,13 +2592,19 @@ core_prepare_for_shutdown(core_handle *spl) platform_assert_status_ok(rc); core_report_unincorporated_memtables(spl); +} - // destroy memtable context (and its memtables) +static void +core_teardown_after_shutdown(core_handle *spl) +{ + // Keep this after checkpoint publication: it supplies the generation cut. memtable_context_deinit(&spl->mt_ctxt); - // release the log + // Keep the log alive through clean-record publication. A later explicit + // tail-sync protocol will own its immutable log metadata separately. if (spl->cfg.use_log) { platform_free(spl->heap_id, spl->log); + spl->log = NULL; } // flush all dirty pages in the cache @@ -2126,14 +2620,21 @@ core_unmount(core_handle *spl) { platform_status rc; - core_prepare_for_shutdown(spl); - rc = core_set_super_block(spl, FALSE, TRUE, FALSE); + /* + * Quiescing leaves the memtable and log contexts live so publication can + * atomically capture the retired generation and root, then record log + * metadata. Teardown is safe regardless of publication success. + */ + core_quiesce_for_shutdown(spl); + rc = core_publish_checkpoint_record(spl, FALSE, TRUE, FALSE); if (!SUCCESS(rc)) { - platform_error_log("core_unmount: failed to update super block: %s\n", + platform_error_log("core_unmount: failed to publish checkpoint record: %s\n", platform_status_to_string(rc)); } + core_teardown_after_shutdown(spl); trunk_context_deinit(&spl->trunk_context); core_destroy_stats(spl); + core_checkpoint_lock_deinit(spl); return rc; } @@ -2143,12 +2644,16 @@ core_unmount(core_handle *spl) void core_destroy(core_handle *spl) { - core_prepare_for_shutdown(spl); + core_quiesce_for_shutdown(spl); + core_teardown_after_shutdown(spl); + /* Records own trunk references and their two dedicated record extents. */ + core_destroy_checkpoint_storage(spl); trunk_context_deinit(&spl->trunk_context); // clear out this splinter table from the meta page. allocator_remove_super_addr(spl->al, spl->id); core_destroy_stats(spl); + core_checkpoint_lock_deinit(spl); } @@ -2181,27 +2686,58 @@ core_print_space_use(platform_log_handle *log_handle, core_handle *spl) /* * core_print_super_block() * - * Fetch a super-block for a running Splinter instance, and print its - * contents. + * Print the fixed checkpoint directory and both independently written record + * slots for a running Splinter instance. */ void core_print_super_block(platform_log_handle *log_handle, core_handle *spl) { - page_handle *super_page; - core_super_block *super = core_get_super_block_if_valid(spl, &super_page); - if (super == NULL) { + core_checkpoint_directory directory; + platform_status rc = core_get_checkpoint_directory(spl, &directory); + if (!SUCCESS(rc)) { + platform_log(log_handle, + "No compatible checkpoint directory for root id %lu\n", + spl->id); + return; + } + + core_checkpoint_records records; + rc = core_load_checkpoint_records(spl, &directory, &records); + if (!SUCCESS(rc)) { + platform_log(log_handle, + "Unable to load checkpoint records for root id %lu: %s\n", + spl->id, + platform_status_to_string(rc)); return; } - platform_log(log_handle, "Superblock root_addr=%lu {\n", super->root_addr); - platform_log(log_handle, "log_meta_addr=%lu\n", super->log_meta_addr); platform_log(log_handle, - "timestamp=%lu, checkpointed=%d, unmounted=%d\n", - super->timestamp, - super->checkpointed, - super->unmounted); + "Checkpoint directory root_id=%lu record_addr=[%lu, %lu] {\n", + directory.table_id, + directory.record_addr[0], + directory.record_addr[1]); + for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { + if (!records.valid[slot]) { + platform_log(log_handle, " record[%lu]: invalid\n", slot); + continue; + } + core_checkpoint_record *record = &records.record[slot]; + platform_log(log_handle, + " record[%lu]: sequence=%lu root_addr=%lu " + "has_incorporated_generation=%d " + "incorporated_generation=%lu " + "timestamp=%lu " + "checkpointed=%d unmounted=%d\n", + slot, + record->sequence, + record->root_addr, + record->has_incorporated_generation, + record->incorporated_generation, + record->timestamp, + record->checkpointed, + record->unmounted); + } platform_log(log_handle, "}\n\n"); - core_release_super_block(spl, super_page); } // clang-format off diff --git a/src/core.h b/src/core.h index 94c10f97..997c0622 100644 --- a/src/core.h +++ b/src/core.h @@ -119,6 +119,10 @@ struct core_handle { trunk_context trunk_context; memtable_context mt_ctxt; + /* Serializes snapshot cuts and A/B checkpoint-record publication. */ + platform_mutex checkpoint_lock; + bool32 checkpoint_lock_initialized; + core_stats *stats; core_compacted_memtable compacted_memtable[MAX_MEMTABLES]; diff --git a/src/log.h b/src/log.h index 325d0764..3dd27172 100644 --- a/src/log.h +++ b/src/log.h @@ -16,16 +16,55 @@ typedef struct log_handle log_handle; typedef struct log_iterator log_iterator; typedef struct log_config log_config; +/* + * Identity of one mini-allocator-backed log stream. It is sufficient for a + * higher-level checkpoint record to describe a stream, but not by itself a + * durable descriptor: core will later persist this information in an + * independently checksummed log-segment record. + */ +typedef struct log_segment_info { + uint64 addr; + uint64 meta_addr; + uint64 magic; +} log_segment_info; + typedef int (*log_write_fn)(log_handle *log, key tuple_key, message data, - uint64 generation); + uint64 memtable_generation, + uint64 leaf_generation); +/* + * Finalize the current append pages into checksummed, immutable pages. + * + * The caller must exclude concurrent log_write() calls until it has taken the + * cache writeback fence that is to make the pages durable. It must also + * serialize concurrent log_seal() calls. seal() itself does not issue I/O or + * a durable barrier. + * + * This is deliberately not a persisted replay-boundary descriptor. A log + * implementation whose metadata can grow after sealing needs an additional + * boundary in the checkpoint record to exclude those later entries. + */ +typedef platform_status (*log_seal_fn)(log_handle *log); +/* + * Detach the current stream after sealing it and prepare a distinct fresh + * stream. The caller must exclude writes throughout the operation and make + * the returned identities durable before allowing writes to the fresh stream. + * A zero sealed.meta_addr means the old stream was empty and discarded. + * Rotation alone does not advance the logical durable-log tail: that requires + * a separate, durable tail/manifest publication by the caller. + */ +typedef platform_status (*log_rotate_fn)(log_handle *log, + log_segment_info *sealed, + log_segment_info *fresh); typedef void (*log_release_fn)(log_handle *log); typedef uint64 (*log_addr_fn)(log_handle *log); typedef uint64 (*log_magic_fn)(log_handle *log); typedef struct log_ops { log_write_fn write; + log_seal_fn seal; + log_rotate_fn rotate; log_release_fn release; log_addr_fn addr; log_addr_fn meta_addr; @@ -38,9 +77,32 @@ struct log_handle { }; static inline int -log_write(log_handle *log, key tuple_key, message data, uint64 generation) +log_write(log_handle *log, + key tuple_key, + message data, + uint64 memtable_generation, + uint64 leaf_generation) +{ + return log->ops->write( + log, tuple_key, data, memtable_generation, leaf_generation); +} + +/* + * Finalize the log's current append pages. See log_seal_fn for the required + * exclusion, boundary, and durability ordering. + */ +static inline platform_status +log_seal(log_handle *log) +{ + return log->ops->seal(log); +} + +static inline platform_status +log_rotate(log_handle *log, + log_segment_info *sealed, + log_segment_info *fresh) { - return log->ops->write(log, tuple_key, data, generation); + return log->ops->rotate(log, sealed, fresh); } static inline void diff --git a/src/memtable.c b/src/memtable.c index 41482e68..e8c8d95e 100644 --- a/src/memtable.c +++ b/src/memtable.c @@ -80,6 +80,20 @@ memtable_end_insert(memtable_context *ctxt) batch_rwlock_unget(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); } +void +memtable_block_inserts(memtable_context *ctxt) +{ + batch_rwlock_get(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); + batch_rwlock_claim_loop(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); + batch_rwlock_lock(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); +} + +void +memtable_unblock_inserts(memtable_context *ctxt) +{ + batch_rwlock_full_unlock(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); +} + static inline bool32 memtable_try_begin_insert_rotation(memtable_context *ctxt) { @@ -97,20 +111,6 @@ memtable_end_insert_rotation(memtable_context *ctxt) batch_rwlock_unclaim(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); } -static inline void -memtable_begin_raw_rotation(memtable_context *ctxt) -{ - batch_rwlock_get(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); - batch_rwlock_claim_loop(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); - batch_rwlock_lock(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); -} - -static inline void -memtable_end_raw_rotation(memtable_context *ctxt) -{ - batch_rwlock_full_unlock(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); -} - void memtable_begin_lookup(memtable_context *ctxt) { @@ -297,7 +297,7 @@ memtable_mark_incorporation_failed(memtable *mt, platform_status status) uint64 memtable_force_finalize(memtable_context *ctxt) { - memtable_begin_raw_rotation(ctxt); + memtable_block_inserts(ctxt); uint64 generation = ctxt->generation; uint64 mt_no = generation % ctxt->cfg.max_memtables; @@ -308,7 +308,7 @@ memtable_force_finalize(memtable_context *ctxt) <= ctxt->cfg.max_memtables); memtable_mark_empty(ctxt); - memtable_end_raw_rotation(ctxt); + memtable_unblock_inserts(ctxt); return current_generation; } @@ -357,12 +357,13 @@ memtable_deinit(cache *cc, memtable *mt) } platform_status -memtable_context_init(memtable_context *ctxt, - platform_heap_id hid, - cache *cc, - memtable_config *cfg, - process_fn process, - void *process_ctxt) +memtable_context_init_at_generation(memtable_context *ctxt, + platform_heap_id hid, + cache *cc, + memtable_config *cfg, + process_fn process, + void *process_ctxt, + uint64 first_generation) { platform_status rc; ZERO_CONTENTS(ctxt); @@ -370,14 +371,22 @@ memtable_context_init(memtable_context *ctxt, ctxt->cfg = *cfg; ctxt->hid = hid; - if (MAX_MEMTABLES < cfg->max_memtables) { - platform_error_log("Configured number of memtables (%lu) exceeds max " - "supported memtables (%d)\n", + if (cfg->max_memtables == 0 || MAX_MEMTABLES < cfg->max_memtables) { + platform_error_log("Configured number of memtables (%lu) must be " + "between 1 and %d\n", cfg->max_memtables, MAX_MEMTABLES); return STATUS_BAD_PARAM; } + if (first_generation > (uint64)-1 - cfg->max_memtables) { + platform_error_log("First memtable generation (%lu) leaves too little " + "space for %lu memtable slots\n", + first_generation, + cfg->max_memtables); + return STATUS_BAD_PARAM; + } + rc = platform_mutex_init( &ctxt->incorporation_mutex, platform_get_module_id(), hid); if (!SUCCESS(rc)) { @@ -394,14 +403,25 @@ memtable_context_init(memtable_context *ctxt, batch_rwlock_init(&ctxt->rwlock); - for (uint64 mt_no = 0; mt_no < cfg->max_memtables; mt_no++) { - uint64 generation = mt_no; + for (uint64 generation_offset = 0; + generation_offset < cfg->max_memtables; + generation_offset++) + { + uint64 generation = first_generation + generation_offset; + uint64 mt_no = generation % cfg->max_memtables; memtable_init(&ctxt->mt[mt_no], cc, cfg, generation); } - ctxt->generation = 0; - ctxt->generation_to_incorporate = 0; - ctxt->generation_retired = (uint64)-1; + ctxt->generation = first_generation; + ctxt->generation_to_incorporate = first_generation; + /* + * A fresh database has no incorporated generation. The UINT64_MAX + * sentinel preserves the existing unsigned generation-ring arithmetic. + * Otherwise, the checkpoint has incorporated every generation before + * first_generation. + */ + ctxt->generation_retired = first_generation == 0 ? (uint64)-1 + : first_generation - 1; ctxt->is_empty = TRUE; @@ -411,6 +431,18 @@ memtable_context_init(memtable_context *ctxt, return STATUS_OK; } +platform_status +memtable_context_init(memtable_context *ctxt, + platform_heap_id hid, + cache *cc, + memtable_config *cfg, + process_fn process, + void *process_ctxt) +{ + return memtable_context_init_at_generation( + ctxt, hid, cc, cfg, process, process_ctxt, 0); +} + void memtable_context_deinit(memtable_context *ctxt) { diff --git a/src/memtable.h b/src/memtable.h index 7c82aadd..bfea7002 100644 --- a/src/memtable.h +++ b/src/memtable.h @@ -159,6 +159,17 @@ memtable_maybe_rotate_and_begin_insert(memtable_context *ctxt, void memtable_end_insert(memtable_context *ctxt); +/* + * Exclude all inserts, including an insert that has already acquired its + * shared insert lock. A checkpoint uses this around sealing the log prefix + * that describes its snapshot. Must be paired with memtable_unblock_inserts. + */ +void +memtable_block_inserts(memtable_context *ctxt); + +void +memtable_unblock_inserts(memtable_context *ctxt); + void memtable_begin_lookup(memtable_context *ctxt); @@ -211,6 +222,24 @@ memtable_context_init(memtable_context *ctxt, process_fn process, void *process_ctxt); +/* + * Initialize the reusable memtable ring at first_generation. Recovery passes + * the checkpoint's incorporated generation plus one; the trunk is understood + * to have incorporated all earlier generations. In particular, slot + * first_generation % max_memtables is the active memtable and the other slots + * represent the following logical generations. first_generation == 0 is the + * fresh-database case and has no incorporated generation. A caller must not + * wrap a checkpoint generation of UINT64_MAX into zero. + */ +platform_status +memtable_context_init_at_generation(memtable_context *ctxt, + platform_heap_id hid, + cache *cc, + memtable_config *cfg, + process_fn process, + void *process_ctxt, + uint64 first_generation); + void memtable_context_deinit(memtable_context *ctxt); diff --git a/src/mini_allocator.c b/src/mini_allocator.c index 1099641f..fa350f78 100644 --- a/src/mini_allocator.c +++ b/src/mini_allocator.c @@ -917,6 +917,247 @@ mini_prefetch(cache *cc, page_type type, uint64 meta_head) mini_for_each(cc, meta_head, type, mini_prefetch_extent, NULL); } +/* + * ----------------------------------------------------------------------------- + * mini_recovery_walk -- Read-only mini-allocator extent enumeration. + * ----------------------------------------------------------------------------- + */ + +static platform_status +mini_recovery_corruption(const char *reason, uint64 addr) +{ + platform_error_log("Malformed mini allocator metadata: %s (addr=%lu).\n", + reason, + addr); + return STATUS_INVALID_STATE; +} + +static bool32 +mini_recovery_valid_geometry(const allocator_config *cfg) +{ + if (cfg == NULL || cfg->io_cfg == NULL || cfg->capacity == 0 + || cfg->io_cfg->page_size == 0 || cfg->io_cfg->extent_size == 0 + || cfg->io_cfg->page_size + < offsetof(mini_meta_hdr, entry_buffer) + || cfg->io_cfg->extent_size < cfg->io_cfg->page_size + || cfg->io_cfg->extent_size % cfg->io_cfg->page_size != 0 + || cfg->capacity % cfg->io_cfg->extent_size != 0) + { + return FALSE; + } + + return TRUE; +} + +static bool32 +mini_recovery_valid_page_addr(const allocator_config *cfg, uint64 addr) +{ + uint64 page_size = cfg->io_cfg->page_size; + + return addr != 0 && addr % page_size == 0 + && addr <= cfg->capacity - page_size; +} + +static bool32 +mini_recovery_valid_extent_addr(const allocator_config *cfg, uint64 addr) +{ + uint64 extent_size = cfg->io_cfg->extent_size; + + return addr != 0 && addr % extent_size == 0 + && addr <= cfg->capacity - extent_size; +} + +static uint64 +mini_recovery_extent_base_addr(const allocator_config *cfg, uint64 addr) +{ + return addr - addr % cfg->io_cfg->extent_size; +} + +static platform_status +mini_recovery_validate_meta_header(const mini_meta_hdr *hdr, + uint64 page_size, + uint64 expected_prev, + uint64 meta_addr) +{ + uint64 first_entry_offset = offsetof(mini_meta_hdr, entry_buffer); + uint64 max_entries = (page_size - first_entry_offset) / sizeof(meta_entry); + if (hdr->prev_meta_addr != expected_prev) { + return mini_recovery_corruption("metadata prev link is not reciprocal", + meta_addr); + } + + if (hdr->num_entries > max_entries) { + return mini_recovery_corruption("metadata entry count exceeds page", + meta_addr); + } + + uint64 expected_pos = first_entry_offset + + (uint64)hdr->num_entries * sizeof(meta_entry); + if (hdr->pos != expected_pos) { + return mini_recovery_corruption("metadata entry position is invalid", + meta_addr); + } + + return STATUS_OK; +} + +static platform_status +mini_recovery_validate_next_meta_addr(const allocator_config *cfg, + uint64 meta_addr, + uint64 next_meta_addr) +{ + if (next_meta_addr == 0) { + return STATUS_OK; + } + + if (!mini_recovery_valid_page_addr(cfg, next_meta_addr)) { + return mini_recovery_corruption("metadata next link is not a page", + next_meta_addr); + } + + uint64 extent_size = cfg->io_cfg->extent_size; + uint64 page_size = cfg->io_cfg->page_size; + uint64 offset = meta_addr % extent_size; + if (offset == extent_size - page_size) { + if (next_meta_addr % extent_size != 0) { + return mini_recovery_corruption( + "metadata extent transition does not start at an extent base", + next_meta_addr); + } + } else if (next_meta_addr != meta_addr + page_size) { + return mini_recovery_corruption( + "metadata pages do not advance contiguously within an extent", + next_meta_addr); + } + + return STATUS_OK; +} + +platform_status +mini_recovery_walk(cache *cc, + uint64 meta_head, + page_type meta_type, + mini_recovery_visit_fn visit, + void *arg) +{ + if (cc == NULL || visit == NULL + || meta_type < PAGE_TYPE_FIRST || meta_type >= NUM_PAGE_TYPES) + { + return STATUS_BAD_PARAM; + } + + allocator *al = cache_get_allocator(cc); + if (al == NULL) { + return STATUS_BAD_PARAM; + } + allocator_config *cfg = allocator_get_config(al); + if (!mini_recovery_valid_geometry(cfg)) { + return STATUS_BAD_PARAM; + } + if (!mini_recovery_valid_page_addr(cfg, meta_head)) { + return mini_recovery_corruption("metadata head is not a page", meta_head); + } + + uint64 meta_addr = meta_head; + uint64 expected_prev = 0; + uint64 prior_meta_extent = (uint64)-1; + uint64 max_meta_pages = cfg->capacity / cfg->io_cfg->page_size; + + for (uint64 page_count = 0; meta_addr != 0; page_count++) { + if (page_count == max_meta_pages) { + return mini_recovery_corruption("metadata chain exceeds disk pages", + meta_addr); + } + + /* + * Report a metadata extent before reading it. The normal mini metadata + * layout is contiguous within an extent, so this is once per physical + * metadata extent; next-link and reciprocal-prev validation below reject + * a malformed chain before it can complete a loop. + */ + uint64 meta_extent = mini_recovery_extent_base_addr(cfg, meta_addr); + if (!mini_recovery_valid_extent_addr(cfg, meta_extent)) { + return mini_recovery_corruption("metadata extent is out of range", + meta_extent); + } + if (meta_extent != prior_meta_extent) { + platform_status rc = visit(meta_extent, + meta_type, + MINI_RECOVERY_EXTENT_METADATA, + MINI_RECOVERY_METADATA_BATCH, + arg); + if (!SUCCESS(rc)) { + return rc; + } + prior_meta_extent = meta_extent; + } + + page_handle *meta_page = cache_get(cc, meta_addr, TRUE, meta_type); + if (meta_page == NULL) { + return STATUS_IO_ERROR; + } + + mini_meta_hdr *hdr = (mini_meta_hdr *)meta_page->data; + platform_status rc = mini_recovery_validate_meta_header( + hdr, cfg->io_cfg->page_size, expected_prev, meta_addr); + if (!SUCCESS(rc)) { + cache_unget(cc, meta_page); + return rc; + } + + uint64 next_meta_addr = hdr->next_meta_addr; + rc = mini_recovery_validate_next_meta_addr( + cfg, meta_addr, next_meta_addr); + if (!SUCCESS(rc)) { + cache_unget(cc, meta_page); + return rc; + } + + meta_entry *entry = first_entry(meta_page); + for (uint64 entry_no = 0; entry_no < hdr->num_entries; entry_no++) { + uint64 batch = meta_entry_batch(entry); + page_type extent_type = meta_entry_type(entry); + uint64 extent_number = + entry->packed >> (META_ENTRY_BATCH_BITS + META_ENTRY_TYPE_BITS); + + if (batch >= MINI_MAX_BATCHES + || extent_type < PAGE_TYPE_FIRST || extent_type >= NUM_PAGE_TYPES + || extent_number == 0 + || extent_number > UINT64_MAX / cfg->io_cfg->extent_size) + { + cache_unget(cc, meta_page); + return mini_recovery_corruption("metadata extent entry is invalid", + meta_addr); + } + + uint64 extent_addr = extent_number * cfg->io_cfg->extent_size; + if (!mini_recovery_valid_extent_addr(cfg, extent_addr)) { + cache_unget(cc, meta_page); + return mini_recovery_corruption("metadata extent is out of range", + extent_addr); + } + + rc = visit(extent_addr, + extent_type, + MINI_RECOVERY_EXTENT_DATA, + batch, + arg); + if (!SUCCESS(rc)) { + cache_unget(cc, meta_page); + return rc; + } + + entry = next_entry(entry); + } + + cache_unget(cc, meta_page); + expected_prev = meta_addr; + meta_addr = next_meta_addr; + } + + return STATUS_OK; +} + /* *----------------------------------------------------------------------------- * mini_meta_cursor -- cursor over a mini_allocator's extent entries. diff --git a/src/mini_allocator.h b/src/mini_allocator.h index 1a7995b7..f596d986 100644 --- a/src/mini_allocator.h +++ b/src/mini_allocator.h @@ -113,6 +113,57 @@ mini_unblock_dec_ref(cache *cc, uint64 meta_head); void mini_prefetch(cache *cc, page_type type, uint64 meta_head); +/* + * mini_recovery_walk -- + * + * Enumerate the physical extents reachable from a finalized mini + * allocator without looking at allocator refcounts. This is the + * discovery primitive used by crash recovery to rebuild those refcounts. + * + * The callback is invoked once for each metadata extent and once for + * every data-extent entry in the on-disk mini metadata stream. Data + * entries are deliberately not deduplicated: repeated entries represent + * repeated references in the mini allocator and are reported exactly as + * recorded. Metadata entries have batch + * MINI_RECOVERY_METADATA_BATCH. + * + * The walker validates the metadata-page chain, page-header bounds, page + * types, batches, and extent addresses before using them. A malformed + * on-disk stream returns STATUS_INVALID_STATE. Callback failures are + * returned unchanged. The walker performs no writes and does not use + * allocator refcounts to decide what to traverse. + * + * The callback is called for a metadata extent before the walker reads a + * page from that extent. That ordering lets an allocator-recovery caller + * establish temporary ownership before cache_get() performs its debug + * allocation checks. + * + * This is a physical enumeration only. Recovering logical reference + * multiplicity, and deduplicating references shared by distinct mini + * allocator roots, remains the responsibility of the higher-level + * trunk/log recovery walker. + */ +typedef enum mini_recovery_extent_kind { + MINI_RECOVERY_EXTENT_METADATA, + MINI_RECOVERY_EXTENT_DATA, +} mini_recovery_extent_kind; + +#define MINI_RECOVERY_METADATA_BATCH ((uint64)-1) + +typedef platform_status (*mini_recovery_visit_fn)( + uint64 extent_addr, + page_type type, + mini_recovery_extent_kind kind, + uint64 batch, + void *arg); + +platform_status +mini_recovery_walk(cache *cc, + uint64 meta_head, + page_type meta_type, + mini_recovery_visit_fn visit, + void *arg); + /* * mini_meta_cursor: a non-blocking cursor over the extent entries of a * finalized mini_allocator. Entries from all batches are interleaved in diff --git a/src/platform_linux/laio.c b/src/platform_linux/laio.c index 87e6c3f5..6988514b 100644 --- a/src/platform_linux/laio.c +++ b/src/platform_linux/laio.c @@ -213,6 +213,35 @@ laio_write(io_handle *ioh, void *buf, uint64 bytes, uint64 addr) return STATUS_IO_ERROR; } +/* + *-------------------------------------------------------------------------- + * laio_durable_barrier -- + * + * Make completed writes durable. Waiting for asynchronous I/O only tells us + * that the kernel accepted the writes; fdatasync() supplies the persistence + * ordering needed by checkpoint publication on buffered files. + *-------------------------------------------------------------------------- + */ +static platform_status +laio_durable_barrier(io_handle *ioh) +{ + laio_handle *io = (laio_handle *)ioh; + int ret; + + do { + ret = fdatasync(io->fd); + } while (ret != 0 && errno == EINTR); + + if (ret == 0) { + return STATUS_OK; + } + + int saved_errno = errno; + platform_error_log("laio_durable_barrier: fdatasync failed: %s\n", + strerror(saved_errno)); + return CONST_STATUS(saved_errno); +} + /* * Accessor method: Return opaque handle to IO-context setup by io_setup(). */ @@ -674,6 +703,7 @@ static io_ops laio_ops = { .async_state_init = laio_async_state_init, .cleanup = laio_cleanup, .wait_all = laio_wait_all, + .durable_barrier = laio_durable_barrier, .print_stats = laio_print_stats, .reset_stats = laio_reset_stats, }; diff --git a/src/platform_linux/platform_io.h b/src/platform_linux/platform_io.h index 1d96bf1a..3ecd12e0 100644 --- a/src/platform_linux/platform_io.h +++ b/src/platform_linux/platform_io.h @@ -78,6 +78,7 @@ typedef platform_status (*io_async_state_init_fn)(io_async_state *state, typedef void (*io_cleanup_fn)(io_handle *io, uint64 count); typedef void (*io_wait_all_fn)(io_handle *io); +typedef platform_status (*io_durable_barrier_fn)(io_handle *io); typedef void (*io_register_thread_fn)(io_handle *io); typedef void (*io_deregister_thread_fn)(io_handle *io); typedef bool32 (*io_max_latency_elapsed_fn)(io_handle *io, timestamp ts); @@ -95,6 +96,7 @@ typedef struct io_ops { io_async_state_init_fn async_state_init; io_cleanup_fn cleanup; io_wait_all_fn wait_all; + io_durable_barrier_fn durable_barrier; io_register_thread_fn register_thread; io_deregister_thread_fn deregister_thread; io_max_latency_elapsed_fn max_latency_elapsed; @@ -204,6 +206,21 @@ io_wait_all(io_handle *io) return io->ops->wait_all(io); } +/* + *-------------------------------------------------------------------------- + * io_durable_barrier + * + * Ensure that writes which completed before this call survive a power loss. + * This is deliberately distinct from io_wait_all(), which only waits for + * asynchronous I/O completion. + *-------------------------------------------------------------------------- + */ +static inline platform_status +io_durable_barrier(io_handle *io) +{ + return io->ops->durable_barrier(io); +} + static inline void io_register_thread(io_handle *io) { diff --git a/src/rc_allocator.c b/src/rc_allocator.c index 70257dc5..7a365fa2 100644 --- a/src/rc_allocator.c +++ b/src/rc_allocator.c @@ -17,7 +17,15 @@ #include "platform_typed_alloc.h" #include "poison.h" -#define RC_ALLOCATOR_META_PAGE_CSUM_SEED (2718281828) +#define RC_ALLOCATOR_META_PAGE_CSUM_SEED (2718281828) +#define RC_ALLOCATOR_CLEAN_STATE_CSUM_SEED (2718281829) + +#define RC_ALLOCATOR_FORMAT_MAGIC (0x534442414C4C4F43ULL) // SDBALLOC +#define RC_ALLOCATOR_FORMAT_VERSION (1) + +#define RC_ALLOCATOR_CLEAN_STATE_MAGIC (0x534442434C45414EULL) // SDBCLEAN +#define RC_ALLOCATOR_CLEAN_STATE_VERSION (1) +#define RC_ALLOCATOR_CLEAN_STATE_SLOTS (2) /* * Base offset from where the allocator starts. Currently hard coded to 0. @@ -35,6 +43,28 @@ */ #define SHOULD_TRACE(addr) (0) // Do not trace anything +/* + * A/B clean-state records live in two fixed extents after the refcount map. + * They are deliberately separate from the sole allocator bootstrap page: a + * torn state update must leave an older valid state record available for the + * next mount or rebuild. + */ +typedef struct ONDISK rc_allocator_clean_state { + uint64 magic; + uint64 format_version; + uint64 sequence; + bool32 clean_shutdown; + checksum128 checksum; +} rc_allocator_clean_state; + +typedef struct rc_allocator_clean_states { + rc_allocator_clean_state state[RC_ALLOCATOR_CLEAN_STATE_SLOTS]; + bool32 valid[RC_ALLOCATOR_CLEAN_STATE_SLOTS]; + bool32 have_newest; + uint64 newest_slot; + bool32 duplicate_sequence; +} rc_allocator_clean_states; + /* *------------------------------------------------------------------------------ * Function declarations and virtual trampolines @@ -231,6 +261,16 @@ rc_allocator_meta_page_checksum(const rc_allocator_meta_page *meta_page) RC_ALLOCATOR_META_PAGE_CSUM_SEED); } +static platform_status +rc_allocator_write_meta_page(rc_allocator *al) +{ + al->meta_page->checksum = rc_allocator_meta_page_checksum(al->meta_page); + return io_write(al->io, + al->meta_page, + al->cfg->io_cfg->page_size, + RC_ALLOCATOR_BASE_OFFSET); +} + static disk_geometry rc_allocator_config_get_disk_geometry(allocator_config *cfg) { @@ -241,6 +281,260 @@ rc_allocator_config_get_disk_geometry(allocator_config *cfg) }; } +static uint64 +rc_allocator_refcount_buffer_size(const allocator_config *cfg) +{ + uint64 buffer_size = cfg->extent_capacity * sizeof(refcount); + return ROUNDUP(buffer_size, cfg->io_cfg->page_size); +} + +static uint64 +rc_allocator_refcount_extent_count(const allocator_config *cfg) +{ + uint64 buffer_size = rc_allocator_refcount_buffer_size(cfg); + return (buffer_size + cfg->io_cfg->extent_size - 1) + / cfg->io_cfg->extent_size; +} + +static uint64 +rc_allocator_clean_state_extent_no(const allocator_config *cfg, uint64 slot) +{ + platform_assert(slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS); + return 1 + rc_allocator_refcount_extent_count(cfg) + slot; +} + +static uint64 +rc_allocator_clean_state_addr(const allocator_config *cfg, uint64 slot) +{ + return rc_allocator_clean_state_extent_no(cfg, slot) + * cfg->io_cfg->extent_size; +} + +static uint64 +rc_allocator_reserved_extent_count(const allocator_config *cfg) +{ + return 1 + rc_allocator_refcount_extent_count(cfg) + + RC_ALLOCATOR_CLEAN_STATE_SLOTS; +} + +static checksum128 +rc_allocator_clean_state_checksum(const rc_allocator_clean_state *state) +{ + return platform_checksum128(state, + offsetof(rc_allocator_clean_state, checksum), + RC_ALLOCATOR_CLEAN_STATE_CSUM_SEED); +} + +static bool32 +rc_allocator_clean_state_is_valid(const rc_allocator_clean_state *state) +{ + return state->magic == RC_ALLOCATOR_CLEAN_STATE_MAGIC + && state->format_version == RC_ALLOCATOR_CLEAN_STATE_VERSION + && state->sequence != 0 + && (state->clean_shutdown == FALSE || state->clean_shutdown == TRUE) + && platform_checksum_is_equal( + state->checksum, rc_allocator_clean_state_checksum(state)); +} + +static platform_status +rc_allocator_read_clean_state(rc_allocator *al, + uint64 slot, + rc_allocator_clean_state *state, + bool32 *valid) +{ + platform_assert(slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS); + platform_assert(sizeof(*state) <= al->cfg->io_cfg->page_size); + + buffer_handle buffer; + platform_status rc = + platform_buffer_init(&buffer, al->cfg->io_cfg->page_size); + if (!SUCCESS(rc)) { + return rc; + } + + void *page = platform_buffer_getaddr(&buffer); + rc = io_read(al->io, + page, + al->cfg->io_cfg->page_size, + rc_allocator_clean_state_addr(al->cfg, slot)); + if (SUCCESS(rc)) { + memcpy(state, page, sizeof(*state)); + *valid = rc_allocator_clean_state_is_valid(state); + } + + platform_status deinit_rc = platform_buffer_deinit(&buffer); + if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { + rc = deinit_rc; + } + return rc; +} + +static platform_status +rc_allocator_write_clean_state(rc_allocator *al, + uint64 slot, + const rc_allocator_clean_state *state, + bool32 durable) +{ + platform_assert(slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS); + platform_assert(sizeof(*state) <= al->cfg->io_cfg->page_size); + + buffer_handle buffer; + platform_status rc = + platform_buffer_init(&buffer, al->cfg->io_cfg->page_size); + if (!SUCCESS(rc)) { + return rc; + } + + void *page = platform_buffer_getaddr(&buffer); + memset(page, 0, al->cfg->io_cfg->page_size); + memcpy(page, state, sizeof(*state)); + rc = io_write(al->io, + page, + al->cfg->io_cfg->page_size, + rc_allocator_clean_state_addr(al->cfg, slot)); + if (SUCCESS(rc) && durable) { + rc = io_durable_barrier(al->io); + } + + platform_status deinit_rc = platform_buffer_deinit(&buffer); + if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { + rc = deinit_rc; + } + return rc; +} + +static platform_status +rc_allocator_load_clean_states(rc_allocator *al, + rc_allocator_clean_states *states) +{ + ZERO_CONTENTS(states); + for (uint64 slot = 0; slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS; slot++) { + platform_status rc = rc_allocator_read_clean_state( + al, slot, &states->state[slot], &states->valid[slot]); + if (!SUCCESS(rc)) { + return rc; + } + + if (!states->valid[slot]) { + continue; + } + if (!states->have_newest + || states->state[states->newest_slot].sequence + < states->state[slot].sequence) + { + states->have_newest = TRUE; + states->newest_slot = slot; + } else if (states->state[states->newest_slot].sequence + == states->state[slot].sequence) + { + states->duplicate_sequence = TRUE; + } + } + return STATUS_OK; +} + +static platform_status +rc_allocator_publish_clean_state(rc_allocator *al, bool32 clean_shutdown) +{ + rc_allocator_clean_states states; + platform_status rc = rc_allocator_load_clean_states(al, &states); + if (!SUCCESS(rc)) { + return rc; + } + + if (states.have_newest + && states.state[states.newest_slot].sequence == UINT64_MAX) + { + return STATUS_LIMIT_EXCEEDED; + } + + uint64 target_slot = states.have_newest ? states.newest_slot ^ 1 : 0; + rc_allocator_clean_state state; + ZERO_CONTENTS(&state); + state.magic = RC_ALLOCATOR_CLEAN_STATE_MAGIC; + state.format_version = RC_ALLOCATOR_CLEAN_STATE_VERSION; + state.sequence = states.have_newest + ? states.state[states.newest_slot].sequence + 1 + : 1; + state.clean_shutdown = clean_shutdown; + state.checksum = rc_allocator_clean_state_checksum(&state); + return rc_allocator_write_clean_state(al, target_slot, &state, TRUE); +} + +static platform_status +rc_allocator_initialize_clean_states(rc_allocator *al) +{ + for (uint64 slot = 0; slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS; slot++) { + rc_allocator_clean_state state; + ZERO_CONTENTS(&state); + state.magic = RC_ALLOCATOR_CLEAN_STATE_MAGIC; + state.format_version = RC_ALLOCATOR_CLEAN_STATE_VERSION; + state.sequence = slot + 1; + state.clean_shutdown = FALSE; + state.checksum = rc_allocator_clean_state_checksum(&state); + + platform_status rc = + rc_allocator_write_clean_state(al, slot, &state, FALSE); + if (!SUCCESS(rc)) { + return rc; + } + } + return io_durable_barrier(al->io); +} + +static void +rc_allocator_record_allocated_extent(rc_allocator *al) +{ + int64 curr_allocated = __sync_add_and_fetch(&al->stats.curr_allocated, 1); + int64 max_allocated = al->stats.max_allocated; + while (curr_allocated > max_allocated) { + __sync_bool_compare_and_swap( + &al->stats.max_allocated, max_allocated, curr_allocated); + max_allocated = al->stats.max_allocated; + } +} + +static platform_status +rc_allocator_recovery_initialize_refcounts(rc_allocator *al) +{ + uint64 reserved_extent_count = rc_allocator_reserved_extent_count(al->cfg); + + if (reserved_extent_count > al->cfg->extent_capacity) { + platform_error_log("Allocator needs %lu reserved extents, but its " + "configured capacity is only %lu extents.\n", + reserved_extent_count, + al->cfg->extent_capacity); + return STATUS_BAD_PARAM; + } + + memset(al->ref_count, + 0, + rc_allocator_refcount_buffer_size(al->cfg)); + + /* + * Extent 0 contains both the allocator meta page and every fixed table + * superblock. The refcount table begins at extent 1; the two extents + * after it hold alternating clean-state records. + */ + for (uint64 extent_no = 0; extent_no < reserved_extent_count; extent_no++) + { + platform_assert(al->ref_count[extent_no] == AL_FREE); + al->ref_count[extent_no] = AL_ONE_REF; + rc_allocator_record_allocated_extent(al); + } + + /* Match a regular mount: the first post-recovery allocation scans at 0. */ + al->hand = 0; + return STATUS_OK; +} + +static bool32 +rc_allocator_recovery_extent_is_reserved(const rc_allocator *al, + uint64 extent_no) +{ + return extent_no < rc_allocator_reserved_extent_count(al->cfg); +} + static platform_status rc_allocator_validate_disk_geometry(rc_allocator *al) { @@ -309,7 +603,9 @@ rc_allocator_init_meta_page(rc_allocator *al) memset(al->meta_page->splinters, INVALID_ALLOCATOR_ROOT_ID, sizeof(al->meta_page->splinters)); - al->meta_page->geometry = rc_allocator_config_get_disk_geometry(al->cfg); + al->meta_page->geometry = rc_allocator_config_get_disk_geometry(al->cfg); + al->meta_page->format_magic = RC_ALLOCATOR_FORMAT_MAGIC; + al->meta_page->format_version = RC_ALLOCATOR_FORMAT_VERSION; return STATUS_OK; } @@ -344,6 +640,15 @@ rc_allocator_valid_config(allocator_config *cfg) return STATUS_BAD_PARAM; } + if (rc_allocator_reserved_extent_count(cfg) > cfg->extent_capacity) { + platform_error_log("Configured allocator has %lu extents, but needs %lu " + "reserved extents for metadata, refcounts, and clean " + "state.\n", + cfg->extent_capacity, + rc_allocator_reserved_extent_count(cfg)); + return STATUS_BAD_PARAM; + } + // Assert: Disk size == (page-size * #-of-pages) if (cfg->capacity != (cfg->io_cfg->page_size * cfg->page_capacity)) { platform_error_log("Configured disk size, %lu bytes, is not an integral" @@ -410,8 +715,7 @@ rc_allocator_init(rc_allocator *al, return rc; } // To ensure alignment always allocate in multiples of page size. - uint64 buffer_size = cfg->extent_capacity * sizeof(refcount); - buffer_size = ROUNDUP(buffer_size, cfg->io_cfg->page_size); + uint64 buffer_size = rc_allocator_refcount_buffer_size(cfg); rc = platform_buffer_init(&al->bh, buffer_size); if (!SUCCESS(rc)) { platform_mutex_destroy(&al->lock); @@ -431,14 +735,42 @@ rc_allocator_init(rc_allocator *al, * Allocate room for the ref counts, use same rounded up size used in buffer * creation. */ - rc_extent_count = (buffer_size + al->cfg->io_cfg->extent_size - 1) - / al->cfg->io_cfg->extent_size; + rc_extent_count = rc_allocator_refcount_extent_count(cfg); for (uint64 i = 0; i < rc_extent_count; i++) { allocator_alloc(&al->super, &addr, PAGE_TYPE_SUPERBLOCK); platform_assert(addr == cfg->io_cfg->extent_size * (i + 1)); } + for (uint64 slot = 0; slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS; slot++) { + allocator_alloc(&al->super, &addr, PAGE_TYPE_SUPERBLOCK); + platform_assert(addr == rc_allocator_clean_state_addr(cfg, slot)); + } + + /* + * Persist the immutable bootstrap layout and both initial false state + * records before returning a newly created allocator. A crash before a + * later clean close therefore enters rebuild recovery rather than trusting + * this freshly initialized refcount map. + */ + rc = rc_allocator_initialize_clean_states(al); + if (!SUCCESS(rc)) { + goto deinit_allocator; + } + rc = rc_allocator_write_meta_page(al); + if (!SUCCESS(rc)) { + goto deinit_allocator; + } + rc = io_durable_barrier(al->io); + if (!SUCCESS(rc)) { + goto deinit_allocator; + } + return STATUS_OK; + +deinit_allocator: + rc_allocator_deinit(al); + ZERO_CONTENTS(al); + return rc; } void @@ -458,12 +790,13 @@ rc_allocator_deinit(rc_allocator *al) * Write the file system to disk *---------------------------------------------------------------------- */ -platform_status -rc_allocator_mount(rc_allocator *al, - allocator_config *cfg, - io_handle *io, - platform_heap_id hid, - platform_module_id mid) +static platform_status +rc_allocator_mount_internal(rc_allocator *al, + allocator_config *cfg, + io_handle *io, + platform_heap_id hid, + platform_module_id mid, + bool32 rebuild_refcounts) { platform_status status; @@ -493,8 +826,7 @@ rc_allocator_mount(rc_allocator *al, platform_assert(cfg->capacity == cfg->io_cfg->page_size * cfg->page_capacity); - uint64 buffer_size = cfg->extent_capacity * sizeof(refcount); - buffer_size = ROUNDUP(buffer_size, cfg->io_cfg->page_size); + uint64 buffer_size = rc_allocator_refcount_buffer_size(cfg); status = platform_buffer_init(&al->bh, buffer_size); if (!SUCCESS(status)) { platform_free(al->heap_id, al->meta_page); @@ -508,45 +840,179 @@ rc_allocator_mount(rc_allocator *al, status = io_read( io, al->meta_page, al->cfg->io_cfg->page_size, RC_ALLOCATOR_BASE_OFFSET); if (!SUCCESS(status)) { - platform_free(al->heap_id, al->meta_page); - platform_buffer_deinit(&al->bh); - platform_mutex_destroy(&al->lock); - return status; + goto deinit_buffer; } status = rc_allocator_validate_disk_geometry(al); if (!SUCCESS(status)) { - platform_free(al->heap_id, al->meta_page); - platform_buffer_deinit(&al->bh); - platform_mutex_destroy(&al->lock); - return status; + goto deinit_buffer; } // validate the checksum of the meta page. checksum128 currChecksum = rc_allocator_meta_page_checksum(al->meta_page); if (!platform_checksum_is_equal(al->meta_page->checksum, currChecksum)) { platform_error_log("Corrupt SplinterDB allocator meta page on mount\n"); - platform_free(al->heap_id, al->meta_page); - platform_buffer_deinit(&al->bh); - platform_mutex_destroy(&al->lock); - return STATUS_BAD_PARAM; + status = STATUS_BAD_PARAM; + goto deinit_buffer; } - // load the ref counts from disk. - status = io_read(io, al->ref_count, buffer_size, cfg->io_cfg->extent_size); - if (!SUCCESS(status)) { - platform_free(al->heap_id, al->meta_page); - platform_buffer_deinit(&al->bh); - platform_mutex_destroy(&al->lock); - return status; + if (al->meta_page->format_magic != RC_ALLOCATOR_FORMAT_MAGIC + || al->meta_page->format_version != RC_ALLOCATOR_FORMAT_VERSION) + { + platform_error_log("Unsupported SplinterDB allocator bootstrap format " + "on mount.\n"); + status = STATUS_BAD_PARAM; + goto deinit_buffer; + } + + if (!rebuild_refcounts) { + rc_allocator_clean_states states; + status = rc_allocator_load_clean_states(al, &states); + if (!SUCCESS(status)) { + goto deinit_buffer; + } + if (!states.have_newest || states.duplicate_sequence + || !states.state[states.newest_slot].clean_shutdown) + { + platform_error_log("Allocator was not cleanly shut down; recovery " + "rebuild is required.\n"); + status = STATUS_INVALID_STATE; + goto deinit_buffer; + } } - for (uint64 i = 0; i < al->cfg->extent_capacity; i++) { - if (al->ref_count[i] != 0) { - al->stats.curr_allocated++; + if (rebuild_refcounts) { + /* Do not leave a stale clean state if recovery itself is interrupted. */ + status = rc_allocator_publish_clean_state(al, FALSE); + if (!SUCCESS(status)) { + goto deinit_buffer; + } + status = rc_allocator_recovery_initialize_refcounts(al); + if (!SUCCESS(status)) { + goto deinit_buffer; + } + al->recovery_in_progress = TRUE; + } else { + // Load the ref counts from disk during a normal, clean mount. + status = io_read(io, al->ref_count, buffer_size, cfg->io_cfg->extent_size); + if (!SUCCESS(status)) { + goto deinit_buffer; + } + + for (uint64 i = 0; i < al->cfg->extent_capacity; i++) { + if (al->ref_count[i] != 0) { + al->stats.curr_allocated++; + } + } + + /* Mark dirty before handing the trusted map to any mutating caller. */ + status = rc_allocator_publish_clean_state(al, FALSE); + if (!SUCCESS(status)) { + goto deinit_buffer; } } return STATUS_OK; + +deinit_buffer: + platform_buffer_deinit(&al->bh); + al->ref_count = NULL; + platform_free(al->heap_id, al->meta_page); + al->meta_page = NULL; + platform_mutex_destroy(&al->lock); + return status; +} + +platform_status +rc_allocator_mount(rc_allocator *al, + allocator_config *cfg, + io_handle *io, + platform_heap_id hid, + platform_module_id mid) +{ + return rc_allocator_mount_internal(al, cfg, io, hid, mid, FALSE); +} + +platform_status +rc_allocator_mount_recovery(rc_allocator *al, + allocator_config *cfg, + io_handle *io, + platform_heap_id hid, + platform_module_id mid) +{ + return rc_allocator_mount_internal(al, cfg, io, hid, mid, TRUE); +} + +platform_status +rc_allocator_rebuild_acquire_extent(rc_allocator *al, uint64 extent_addr) +{ + if (!al->recovery_in_progress) { + platform_error_log("Cannot acquire allocator extent while recovery is " + "not in progress.\n"); + return STATUS_INVALID_STATE; + } + + uint64 extent_size = al->cfg->io_cfg->extent_size; + if (extent_addr >= al->cfg->capacity || extent_addr % extent_size != 0) { + platform_error_log("Invalid allocator recovery extent address %lu.\n", + extent_addr); + return STATUS_BAD_PARAM; + } + + uint64 extent_no = rc_allocator_extent_number(al, extent_addr); + platform_assert(extent_no < al->cfg->extent_capacity); + if (rc_allocator_recovery_extent_is_reserved(al, extent_no)) { + platform_error_log("Cannot rebuild ownership of reserved allocator " + "extent %lu.\n", + extent_no); + return STATUS_BAD_PARAM; + } + + while (TRUE) { + refcount old_ref = __atomic_load_n(&al->ref_count[extent_no], + __ATOMIC_RELAXED); + if (old_ref == (refcount)-1) { + platform_error_log("Allocator recovery refcount overflow for extent " + "%lu.\n", + extent_no); + return STATUS_LIMIT_EXCEEDED; + } + + refcount new_ref = + old_ref == AL_FREE ? AL_ONE_REF : old_ref + 1; + if (!__sync_bool_compare_and_swap( + &al->ref_count[extent_no], old_ref, new_ref)) + { + continue; + } + + if (old_ref == AL_FREE) { + rc_allocator_record_allocated_extent(al); + } + return STATUS_OK; + } +} + +void +rc_allocator_rebuild_finish(rc_allocator *al) +{ + platform_assert(al != NULL); + platform_assert(al->recovery_in_progress); + + /* + * Intentionally no I/O here. A rebuilt map is durable only after a later + * clean unmount; another crash before then simply rebuilds it again. + */ + al->recovery_in_progress = FALSE; +} + +void +rc_allocator_abort_recovery(rc_allocator *al) +{ + platform_assert(al != NULL); + platform_assert(al->recovery_in_progress); + + rc_allocator_deinit(al); + ZERO_CONTENTS(al); } @@ -555,12 +1021,32 @@ rc_allocator_unmount(rc_allocator *al) { platform_status status; + if (al->recovery_in_progress) { + platform_error_log("Discarding incomplete allocator recovery instead of " + "persisting its partial refcount map.\n"); + rc_allocator_abort_recovery(al); + return; + } + // persist the ref counts upon unmount. - uint64 buffer_size = al->cfg->extent_capacity * sizeof(refcount); + uint64 buffer_size = rc_allocator_refcount_buffer_size(al->cfg); uint32 io_size = ROUNDUP(buffer_size, al->cfg->io_cfg->page_size); status = io_write(al->io, al->ref_count, io_size, al->cfg->io_cfg->extent_size); platform_assert_status_ok(status); + + /* + * This is the sole normal persistence point for the allocator map. The + * checkpoint record was already made durable by core_unmount(); do not + * advertise a clean allocator snapshot until this write has crossed the + * device durability boundary as well. + */ + status = io_durable_barrier(al->io); + platform_assert_status_ok(status); + + /* Publish clean permission in the alternate durable state record. */ + status = rc_allocator_publish_clean_state(al, TRUE); + platform_assert_status_ok(status); rc_allocator_deinit(al); } @@ -686,12 +1172,7 @@ rc_allocator_alloc_super_addr(rc_allocator *al, // assign the first available slot and update the on disk metadata. al->meta_page->splinters[idx] = allocator_root_id; *addr = (1 + idx) * al->cfg->io_cfg->page_size; - al->meta_page->checksum = - rc_allocator_meta_page_checksum(al->meta_page); - platform_status io_status = io_write(al->io, - al->meta_page, - al->cfg->io_cfg->page_size, - RC_ALLOCATOR_BASE_OFFSET); + platform_status io_status = rc_allocator_write_meta_page(al); platform_assert_status_ok(io_status); status = STATUS_OK; break; @@ -715,12 +1196,7 @@ rc_allocator_remove_super_addr(rc_allocator *al, */ if (al->meta_page->splinters[idx] == allocator_root_id) { al->meta_page->splinters[idx] = INVALID_ALLOCATOR_ROOT_ID; - al->meta_page->checksum = - rc_allocator_meta_page_checksum(al->meta_page); - platform_status status = io_write(al->io, - al->meta_page, - al->cfg->io_cfg->page_size, - RC_ALLOCATOR_BASE_OFFSET); + platform_status status = rc_allocator_write_meta_page(al); platform_assert_status_ok(status); platform_mutex_unlock(&al->lock); return; diff --git a/src/rc_allocator.h b/src/rc_allocator.h index 06ed7256..079e565a 100644 --- a/src/rc_allocator.h +++ b/src/rc_allocator.h @@ -33,7 +33,14 @@ *---------------------------------------------------------------------- */ typedef struct ONDISK rc_allocator_meta_page { - disk_geometry geometry; + disk_geometry geometry; + /* + * Identifies the immutable bootstrap layout. In particular, it proves + * that the two fixed clean-state extents after the refcount map are owned + * by this allocator rather than by an older on-disk format. + */ + uint64 format_magic; + uint64 format_version; allocator_root_id splinters[RC_ALLOCATOR_MAX_ROOT_IDS]; checksum128 checksum; } rc_allocator_meta_page; @@ -76,6 +83,13 @@ typedef struct rc_allocator { platform_mutex lock; platform_heap_id heap_id; + /* + * True between rc_allocator_mount_recovery() and either + * rc_allocator_rebuild_finish() or rc_allocator_abort_recovery(). An + * incomplete rebuilt map must never be written back to disk. + */ + bool32 recovery_in_progress; + // Stats -- not distributed for now rc_allocator_stats stats; } rc_allocator; @@ -90,6 +104,14 @@ rc_allocator_init(rc_allocator *al, void rc_allocator_deinit(rc_allocator *al); +/* + * Normal mount accepts only a durable clean-state record and publishes an + * unclean state record before returning. Those records occupy two fixed, + * alternating extents after the persisted refcount map, so normal lifecycle + * operations never rewrite the sole allocator bootstrap page. + * STATUS_INVALID_STATE means callers must use the recovery-rebuild path + * instead of trusting the persisted refcount map. + */ platform_status rc_allocator_mount(rc_allocator *al, allocator_config *cfg, @@ -97,6 +119,43 @@ rc_allocator_mount(rc_allocator *al, platform_heap_id hid, platform_module_id mid); +/* + * Mount the allocator for crash recovery. This validates and retains the + * allocator metadata page, but deliberately ignores the persisted refcount + * table. The caller must rebuild the in-memory table from durable objects, + * then call rc_allocator_rebuild_finish() before using normal allocator + * lifecycle operations. + */ +platform_status +rc_allocator_mount_recovery(rc_allocator *al, + allocator_config *cfg, + io_handle *io, + platform_heap_id hid, + platform_module_id mid); + +/* + * Add one logical ownership reference for an extent while rebuilding a + * recovery map. The first reference establishes the allocator's nonzero + * allocation floor (AL_ONE_REF); later references increment it normally. + * extent_addr must be the base address of a non-reserved allocator extent. + */ +platform_status +rc_allocator_rebuild_acquire_extent(rc_allocator *al, uint64 extent_addr); + +/* + * Complete a successful rebuild without performing I/O. A later normal + * rc_allocator_unmount() may persist the rebuilt table on clean shutdown. + */ +void +rc_allocator_rebuild_finish(rc_allocator *al); + +/* + * Discard a partially rebuilt recovery map. Unlike rc_allocator_unmount(), + * this never writes allocator state to disk. + */ +void +rc_allocator_abort_recovery(rc_allocator *al); + platform_status rc_allocator_read_disk_geometry(const char *filename, disk_geometry *geometry); diff --git a/src/shard_log.c b/src/shard_log.c index c0eee131..9c1fd016 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -25,8 +25,19 @@ static uint64 shard_log_magic_idx = 0; -int -shard_log_write(log_handle *log, key tuple_key, message msg, uint64 generation); +int shard_log_write(log_handle *log, + key tuple_key, + message msg, + uint64 memtable_generation, + uint64 leaf_generation); +platform_status +shard_log_seal(log_handle *log); +platform_status +shard_log_rotate(log_handle *log, + log_segment_info *sealed, + log_segment_info *fresh); +void +shard_log_release(log_handle *log); uint64 shard_log_addr(log_handle *log); uint64 @@ -36,6 +47,9 @@ shard_log_magic(log_handle *log); static log_ops shard_log_ops = { .write = shard_log_write, + .seal = shard_log_seal, + .rotate = shard_log_rotate, + .release = shard_log_release, .addr = shard_log_addr, .meta_addr = shard_log_meta_addr, .magic = shard_log_magic, @@ -93,6 +107,9 @@ page_handle * shard_log_alloc(shard_log *log, uint64 *next_extent) { uint64 addr = mini_alloc_page(&log->mini, 0, next_extent); + if (addr == 0) { + return NULL; + } return cache_alloc(log->cc, addr, PAGE_TYPE_LOG); } @@ -142,7 +159,15 @@ shard_log_zap(shard_log *log) thread_data->offset = 0; } - mini_dec_ref(cc, log->meta_head, PAGE_TYPE_LOG); + if (log->meta_head != 0) { + /* Drop unused per-batch reserves before releasing the mini root. */ + mini_release(&log->mini); + refcount ref = mini_dec_ref(cc, log->meta_head, PAGE_TYPE_LOG); + platform_assert(ref == 0); + log->meta_head = 0; + log->addr = 0; + log->has_pages = FALSE; + } } /* @@ -152,11 +177,12 @@ shard_log_zap(shard_log *log) * ------------------------------------------------------------------------- */ struct ONDISK log_entry { - uint64 generation; + uint64 memtable_generation; + uint64 leaf_generation; ondisk_tuple tuple; }; -#define INVALID_GENERATION ((uint64) - 1) +#define INVALID_LOG_GENERATION ((uint64)-1) static key log_entry_key(log_entry *le) @@ -200,7 +226,19 @@ static bool32 terminal_log_entry(shard_log_config *cfg, char *page, log_entry *le) { return page + shard_log_page_size(cfg) - (char *)le < sizeof(log_entry) - || le->generation == INVALID_GENERATION; + || le->memtable_generation == INVALID_LOG_GENERATION; +} + +static inline void +log_entry_set_terminal(log_entry *le) +{ + /* + * terminal_log_entry() tests memtable_generation. Mark both generation + * fields invalid so the terminator cannot be confused with a record by + * diagnostics or a future format validator. + */ + le->leaf_generation = INVALID_LOG_GENERATION; + le->memtable_generation = INVALID_LOG_GENERATION; } static log_entry * @@ -217,19 +255,29 @@ get_new_page_for_thread(shard_log *log, uint64 next_extent; *page = shard_log_alloc(log, &next_extent); + if (*page == NULL) { + return -1; + } thread_data->addr = (*page)->disk_addr; shard_log_hdr *hdr = (shard_log_hdr *)(*page)->data; hdr->magic = log->magic; hdr->next_extent_addr = next_extent; hdr->num_entries = 0; thread_data->offset = sizeof(shard_log_hdr); + log->has_pages = TRUE; return 0; } int -shard_log_write(log_handle *logh, key tuple_key, message msg, uint64 generation) +shard_log_write(log_handle *logh, + key tuple_key, + message msg, + uint64 memtable_generation, + uint64 leaf_generation) { debug_assert(key_is_user_key(tuple_key)); + debug_assert(memtable_generation != INVALID_LOG_GENERATION); + debug_assert(leaf_generation != INVALID_LOG_GENERATION); shard_log *log = (shard_log *)logh; cache *cc = log->cc; @@ -290,7 +338,7 @@ shard_log_write(log_handle *logh, key tuple_key, message msg, uint64 generation) if (free_space < new_entry_size) { if (sizeof(log_entry) <= free_space) { - cursor->generation = INVALID_GENERATION; + log_entry_set_terminal(cursor); } hdr->checksum = shard_log_checksum(log->cfg, page); @@ -309,7 +357,8 @@ shard_log_write(log_handle *logh, key tuple_key, message msg, uint64 generation) hdr = (shard_log_hdr *)page->data; } - cursor->generation = generation; + cursor->memtable_generation = memtable_generation; + cursor->leaf_generation = leaf_generation; copy_tuple_to_ondisk_tuple(&cursor->tuple, tuple_key, msg); hdr->num_entries++; @@ -332,6 +381,152 @@ shard_log_write(log_handle *logh, key tuple_key, message msg, uint64 generation) return 0; } +/* + * shard_log_seal -- + * + * Finalize every currently active per-thread append page. This is + * deliberately bounded by MAX_THREADS: it does not walk the historical + * log. A final terminal record (where there is room) and checksum make + * each page readable by shard_log_iterator_init(), then clearing the + * append cursor ensures a subsequent writer allocates a new page instead + * of changing the sealed one. + * + * The caller must prevent concurrent shard_log_write() and seal calls. + * In particular, thread_data is otherwise only protected by the + * per-thread writer convention, not by a log-wide lock. This function + * intentionally does not issue writeback: after establishing that + * exclusion, the caller takes cache_writeback_fence(), followed by a + * durable barrier. + */ +platform_status +shard_log_seal(log_handle *logh) +{ + shard_log *log = (shard_log *)logh; + cache *cc = log->cc; + + for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { + shard_log_thread_data *thread_data = + shard_log_get_thread_data(log, thr_i); + uint64 addr = thread_data->addr; + if (addr == SHARD_UNMAPPED) { + continue; + } + + page_handle *page = cache_get(cc, addr, TRUE, PAGE_TYPE_LOG); + uint64 wait = 1; + while (!cache_try_claim(cc, page)) { + cache_unget(cc, page); + platform_sleep_ns(wait); + wait = wait > 1024 ? wait : 2 * wait; + page = cache_get(cc, addr, TRUE, PAGE_TYPE_LOG); + } + cache_lock(cc, page); + + debug_assert(thread_data->addr == addr); + debug_assert(thread_data->offset >= sizeof(shard_log_hdr)); + debug_assert(thread_data->offset <= shard_log_page_size(log->cfg)); + + shard_log_hdr *hdr = (shard_log_hdr *)page->data; + log_entry *cursor = + (log_entry *)(page->data + thread_data->offset); + uint64 free_space = shard_log_page_size(log->cfg) - thread_data->offset; + if (sizeof(log_entry) <= free_space) { + log_entry_set_terminal(cursor); + } + hdr->checksum = shard_log_checksum(log->cfg, page); + cache_mark_dirty(cc, page); + + cache_unlock(cc, page); + cache_unclaim(cc, page); + cache_unget(cc, page); + + /* Subsequent writes must allocate a new append page. */ + thread_data->addr = SHARD_UNMAPPED; + thread_data->offset = 0; + } + + return STATUS_OK; +} + +/* + * Detach a sealed stream from the live log. The original allocation reference + * on its metadata extent is transferred to sealed; an eventual durable + * descriptor must own that reference. A fresh mini allocator is prepared + * before the old stream is sealed, so core can publish the fresh identity + * while inserts remain excluded. This only prepares a physical boundary; it + * does not publish or advance the logical durable-log tail. + */ +platform_status +shard_log_rotate(log_handle *logh, + log_segment_info *sealed, + log_segment_info *fresh_info) +{ + shard_log *log = (shard_log *)logh; + shard_log fresh; + + platform_assert(sealed != NULL); + platform_assert(fresh_info != NULL); + ZERO_CONTENTS(sealed); + ZERO_CONTENTS(fresh_info); + + platform_status rc = shard_log_init(&fresh, log->cc, log->cfg); + if (!SUCCESS(rc)) { + return rc; + } + + rc = shard_log_seal(logh); + if (!SUCCESS(rc)) { + shard_log_zap(&fresh); + return rc; + } + + /* No future allocation may use the old mini allocator. */ + mini_release(&log->mini); + + if (log->has_pages) { + *sealed = (log_segment_info){ + .addr = log->addr, + .meta_addr = log->meta_head, + .magic = log->magic, + }; + } else { + /* An empty segment has no descriptor and no retained ownership. */ + refcount ref = mini_dec_ref(log->cc, log->meta_head, PAGE_TYPE_LOG); + platform_assert(ref == 0); + } + + log->mini = fresh.mini; + log->addr = fresh.addr; + log->meta_head = fresh.meta_head; + log->magic = fresh.magic; + log->has_pages = FALSE; + memcpy(log->thread_data, fresh.thread_data, sizeof(log->thread_data)); + + *fresh_info = (log_segment_info){ + .addr = log->addr, + .meta_addr = log->meta_head, + .magic = log->magic, + }; + return STATUS_OK; +} + +void +shard_log_segment_discard(cache *cc, + const log_segment_info *segment) +{ + if (segment->meta_addr == 0) { + return; + } + refcount ref = mini_dec_ref(cc, segment->meta_addr, PAGE_TYPE_LOG); + platform_assert(ref == 0); +} + +void +shard_log_release(log_handle *logh) +{ + shard_log_zap((shard_log *)logh); +} + uint64 shard_log_addr(log_handle *logh) { @@ -374,7 +569,20 @@ shard_log_compare(const void *p1, const void *p2, void *unused) { log_entry **le1 = (log_entry **)p1; log_entry **le2 = (log_entry **)p2; - return (*le1)->generation - (*le2)->generation; + + if ((*le1)->memtable_generation < (*le2)->memtable_generation) { + return -1; + } + if ((*le1)->memtable_generation > (*le2)->memtable_generation) { + return 1; + } + if ((*le1)->leaf_generation < (*le2)->leaf_generation) { + return -1; + } + if ((*le1)->leaf_generation > (*le2)->leaf_generation) { + return 1; + } + return 0; } log_handle * @@ -527,6 +735,16 @@ shard_log_iterator_curr(iterator *itorh, key *curr_key, message *msg) *msg = log_entry_message(itor->cc, itor->entries[itor->pos]); } +void +shard_log_iterator_curr_generations(shard_log_iterator *itor, + uint64 *memtable_generation, + uint64 *leaf_generation) +{ + platform_assert(itor->pos < itor->num_entries); + *memtable_generation = itor->entries[itor->pos]->memtable_generation; + *leaf_generation = itor->entries[itor->pos]->leaf_generation; +} + bool32 shard_log_iterator_can_prev(iterator *itorh) { @@ -597,11 +815,12 @@ shard_log_print(shard_log *log) le = log_entry_next(le)) { platform_default_log( - "%s -- %s%s : %lu\n", + "%s -- %s%s : memtable=%lu leaf=%lu\n", key_string(dcfg, log_entry_key(le)), log_entry_message_is_blob(le) ? "(blob) " : "", message_string(dcfg, log_entry_message(cc, le)), - le->generation); + le->memtable_generation, + le->leaf_generation); } } cache_unget(cc, page); diff --git a/src/shard_log.h b/src/shard_log.h index 6459d78d..3e024d14 100644 --- a/src/shard_log.h +++ b/src/shard_log.h @@ -46,6 +46,8 @@ typedef struct shard_log { uint64 addr; uint64 meta_head; uint64 magic; + /* Set once any log page has been allocated; survives sealing. */ + bool32 has_pages; } shard_log; typedef struct log_entry log_entry; @@ -76,9 +78,23 @@ typedef struct ONDISK shard_log_hdr { platform_status shard_log_init(shard_log *log, cache *cc, shard_log_config *cfg); +platform_status +shard_log_rotate(log_handle *log, + log_segment_info *sealed, + log_segment_info *fresh); + void shard_log_zap(shard_log *log); +/* + * Drop the external mini-allocator ownership transferred to a detached + * segment. It is for failed rotation cleanup and tests; persisted segment + * descriptors will eventually own and release this reference instead. + */ +void +shard_log_segment_discard(cache *cc, + const log_segment_info *segment); + platform_status shard_log_iterator_init(cache *cc, shard_log_config *cfg, @@ -90,6 +106,15 @@ shard_log_iterator_init(cache *cc, void shard_log_iterator_deinit(platform_heap_id hid, shard_log_iterator *itor); +/* + * Return the generation metadata of the current record. The caller must + * first establish that the iterator has a current record. + */ +void +shard_log_iterator_curr_generations(shard_log_iterator *itor, + uint64 *memtable_generation, + uint64 *leaf_generation); + void shard_log_config_init(shard_log_config *log_cfg, cache_config *cache_cfg, diff --git a/src/splinterdb.c b/src/splinterdb.c index b8614668..c40dc8c3 100644 --- a/src/splinterdb.c +++ b/src/splinterdb.c @@ -498,7 +498,8 @@ splinterdb_create_or_open(const splinterdb_config *kvs_cfg, // IN deinit_cache: clockcache_deinit(&kvs->cache_handle); deinit_allocator: - rc_allocator_unmount(&kvs->allocator_handle); + /* Initialization/open failed before a successful clean core shutdown. */ + rc_allocator_deinit(&kvs->allocator_handle); deinit_system: task_system_deinit(&kvs->task_sys); deinit_iohandle: @@ -578,7 +579,13 @@ splinterdb_close(splinterdb **kvs_in) // IN } io_wait_all(kvs->io_handle); clockcache_deinit(&kvs->cache_handle); - rc_allocator_unmount(&kvs->allocator_handle); + if (SUCCESS(status)) { + /* This writes the map and only then publishes allocator clean state. */ + rc_allocator_unmount(&kvs->allocator_handle); + } else { + /* Keep the durable clean marker cleared; the next open must rebuild. */ + rc_allocator_deinit(&kvs->allocator_handle); + } task_system_deinit(&kvs->task_sys); io_handle_destroy(kvs->io_handle); diff --git a/src/trunk.c b/src/trunk.c index 133139a5..d24c17ae 100644 --- a/src/trunk.c +++ b/src/trunk.c @@ -2541,6 +2541,46 @@ trunk_read_end(trunk_context *context) batch_rwlock_unget(&context->root_lock, 0); } +/* + * Capture the root address and acquire its allocator reference while the root + * lock prevents COW publication from dropping the live reference. This is + * deliberately metadata-only: checkpointing must not fault the root page just + * to take a snapshot. + */ +platform_status +trunk_snapshot_acquire(trunk_context *context, trunk_snapshot *snapshot) +{ + snapshot->root_addr = 0; + + trunk_read_begin(context); + if (context->root != NULL) { + snapshot->root_addr = context->root->ref.addr; + trunk_inc_ref(context->al, snapshot->root_addr); + } + trunk_read_end(context); + + return STATUS_OK; +} + +platform_status +trunk_snapshot_release(trunk_context *context, trunk_snapshot *snapshot) +{ + if (snapshot->root_addr == 0) { + return STATUS_OK; + } + + platform_status rc = trunk_dec_ref(context->cfg, + context->hid, + context->cc, + context->al, + context->ts, + snapshot->root_addr); + if (SUCCESS(rc)) { + snapshot->root_addr = 0; + } + return rc; +} + platform_status trunk_init_root_handle(trunk_context *context, trunk_ondisk_node_handle *handle) { @@ -6814,8 +6854,11 @@ trunk_context_clone(trunk_context *dst, trunk_context *src) platform_status trunk_make_durable(trunk_context *context) { - cache_flush(context->cc); - return STATUS_OK; + platform_status rc = cache_writeback_fence(context->cc); + if (!SUCCESS(rc)) { + return rc; + } + return cache_durable_barrier(context->cc); } /************************************ diff --git a/src/trunk.h b/src/trunk.h index 7df20d09..a48dbab0 100644 --- a/src/trunk.h +++ b/src/trunk.h @@ -174,6 +174,15 @@ typedef struct trunk_context { incorporation_tasks tasks; } trunk_context; +/* + * An owned, point-in-time reference to a COW trunk root. The reference keeps + * the root (and therefore its reachable subtree) alive until it is either + * released or transferred to a durable checkpoint record. + */ +typedef struct trunk_snapshot { + uint64 root_addr; +} trunk_snapshot; + typedef struct trunk_ondisk_node_handle { cache *cc; page_handle *header_page; @@ -223,6 +232,14 @@ trunk_context_deinit(trunk_context *context); platform_status trunk_context_clone(trunk_context *dst, trunk_context *src); +/* Capture an owned reference to the current COW root without reading it. */ +platform_status +trunk_snapshot_acquire(trunk_context *context, trunk_snapshot *snapshot); + +/* Drop an owned snapshot reference that was not published. */ +platform_status +trunk_snapshot_release(trunk_context *context, trunk_snapshot *snapshot); + /* Make a trunk durable */ platform_status trunk_make_durable(trunk_context *context); diff --git a/tests/functional/btree_test.c b/tests/functional/btree_test.c index 2511e615..139db99e 100644 --- a/tests/functional/btree_test.c +++ b/tests/functional/btree_test.c @@ -17,6 +17,7 @@ #include "rc_allocator.h" #include "cache.h" #include "clockcache.h" +#include "mini_allocator.h" #include "random.h" #include "task.h" @@ -70,6 +71,198 @@ test_btree_process_noop(void *arg, uint64 generation) // really a no-op } +static platform_status +test_memtable_generation_init(cache *cc, + test_btree_config *cfg, + platform_heap_id hid) +{ + const uint64 first_generation = 17; + const uint64 max_memtables = 4; + memtable_config mt_cfg = *cfg->mt_cfg; + memtable_context mt_ctxt; + + mt_cfg.max_memtables = max_memtables; + + platform_status rc = memtable_context_init( + &mt_ctxt, hid, cc, &mt_cfg, test_btree_process_noop, NULL); + if (!SUCCESS(rc)) { + return rc; + } + + if (mt_ctxt.generation != 0 || mt_ctxt.generation_to_incorporate != 0 + || mt_ctxt.generation_retired != (uint64)-1) + { + platform_error_log("memtable fresh initialization has unexpected " + "generation state\n"); + rc = STATUS_TEST_FAILED; + goto deinit_fresh; + } + + for (uint64 generation = 0; generation < max_memtables; generation++) { + uint64 mt_no = generation % max_memtables; + if (mt_ctxt.mt[mt_no].generation != generation) { + platform_error_log("memtable fresh initialization put generation " + "%lu in the wrong slot\n", + generation); + rc = STATUS_TEST_FAILED; + goto deinit_fresh; + } + } + +deinit_fresh: + memtable_context_deinit(&mt_ctxt); + if (!SUCCESS(rc)) { + return rc; + } + + rc = memtable_context_init_at_generation(&mt_ctxt, + hid, + cc, + &mt_cfg, + test_btree_process_noop, + NULL, + first_generation); + if (!SUCCESS(rc)) { + return rc; + } + + if (mt_ctxt.generation != first_generation + || mt_ctxt.generation_to_incorporate != first_generation + || mt_ctxt.generation_retired != first_generation - 1) + { + platform_error_log("memtable recovery initialization has unexpected " + "generation state\n"); + rc = STATUS_TEST_FAILED; + goto deinit_recovery; + } + + for (uint64 generation = first_generation; + generation < first_generation + max_memtables; + generation++) + { + uint64 mt_no = generation % max_memtables; + if (mt_ctxt.mt[mt_no].generation != generation) { + platform_error_log("memtable recovery initialization put generation " + "%lu in the wrong slot\n", + generation); + rc = STATUS_TEST_FAILED; + goto deinit_recovery; + } + } + + memtable_block_inserts(&mt_ctxt); + memtable_unblock_inserts(&mt_ctxt); + +deinit_recovery: + memtable_context_deinit(&mt_ctxt); + if (SUCCESS(rc)) { + platform_default_log("btree_test: memtable generation init test passed\n"); + } + return rc; +} + +typedef struct test_mini_recovery_walk_state { + uint64 metadata_visits; + uint64 data_visits; + uint64 metadata_extent; + uint64 data_extent; + bool32 fail_data_visit; +} test_mini_recovery_walk_state; + +static platform_status +test_mini_recovery_walk_visit(uint64 extent_addr, + page_type type, + mini_recovery_extent_kind kind, + uint64 batch, + void *arg) +{ + test_mini_recovery_walk_state *state = arg; + + if (type != PAGE_TYPE_MISC) { + return STATUS_TEST_FAILED; + } + + if (kind == MINI_RECOVERY_EXTENT_METADATA) { + if (batch != MINI_RECOVERY_METADATA_BATCH) { + return STATUS_TEST_FAILED; + } + state->metadata_visits++; + state->metadata_extent = extent_addr; + return STATUS_OK; + } + + if (kind != MINI_RECOVERY_EXTENT_DATA || batch != 0) { + return STATUS_TEST_FAILED; + } + + state->data_visits++; + state->data_extent = extent_addr; + return state->fail_data_visit ? STATUS_TEST_FAILED : STATUS_OK; +} + +static platform_status +test_mini_recovery_walk(cache *cc) +{ + allocator *al = cache_get_allocator(cc); + mini_allocator mini; + test_mini_recovery_walk_state state; + uint64 meta_head = 0; + uint64 data_extent = 0; + platform_status rc = + allocator_alloc(al, &meta_head, PAGE_TYPE_MISC); + if (!SUCCESS(rc)) { + return rc; + } + + mini_init(&mini, cc, meta_head, 0, 1, PAGE_TYPE_MISC); + data_extent = mini_alloc_extent(&mini, 0, NULL); + if (data_extent == 0) { + rc = STATUS_NO_SPACE; + goto cleanup; + } + + ZERO_CONTENTS(&state); + rc = mini_recovery_walk( + cc, meta_head, PAGE_TYPE_MISC, test_mini_recovery_walk_visit, &state); + if (!SUCCESS(rc)) { + goto cleanup; + } + if (state.metadata_visits != 1 || state.data_visits != 1 + || state.metadata_extent != meta_head + || state.data_extent != data_extent) + { + platform_error_log("mini recovery walker returned unexpected extents\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + ZERO_CONTENTS(&state); + state.fail_data_visit = TRUE; + rc = mini_recovery_walk( + cc, meta_head, PAGE_TYPE_MISC, test_mini_recovery_walk_visit, &state); + if (!STATUS_IS_EQ(rc, STATUS_TEST_FAILED) || state.metadata_visits != 1 + || state.data_visits != 1) + { + platform_error_log("mini recovery walker did not propagate callback " + "failure\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc = STATUS_OK; + +cleanup: + mini_release(&mini); + refcount ref = mini_dec_ref(cc, meta_head, PAGE_TYPE_MISC); + if (ref != 0 && SUCCESS(rc)) { + platform_error_log("mini recovery walker left an unexpected mini ref\n"); + rc = STATUS_TEST_FAILED; + } + if (SUCCESS(rc)) { + platform_default_log("btree_test: mini recovery walker test passed\n"); + } + return rc; +} + test_memtable_context * test_memtable_context_create(cache *cc, test_btree_config *cfg, @@ -2258,6 +2451,12 @@ btree_test(int argc, char *argv[]) platform_assert_status_ok(rc); cache *ccp = (cache *)cc; + rc = test_memtable_generation_init(ccp, &test_cfg, hid); + platform_assert_status_ok(rc); + + rc = test_mini_recovery_walk(ccp); + platform_assert_status_ok(rc); + uint64 max_tuples_per_memtable = test_cfg.mt_cfg->max_extents_per_memtable * cache_config_extent_size((cache_config *)&system_cfg.cache_cfg) / 3 diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index ad4b3b27..d75267d3 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -90,6 +90,688 @@ cache_test_alloc_extents(cache *cc, return rc; } +static platform_status +cache_test_fill_page(cache *cc, uint64 addr, uint64 page_size, uint8 value) +{ + page_handle *page = cache_get(cc, addr, TRUE, PAGE_TYPE_MISC); + if (!cache_try_claim(cc, page)) { + cache_unget(cc, page); + return STATUS_BUSY; + } + + cache_lock(cc, page); + memset(page->data, value, page_size); + cache_unlock(cc, page); + cache_unclaim(cc, page); + cache_unget(cc, page); + return STATUS_OK; +} + +static platform_status +cache_test_verify_disk_page(cache *cc, + uint64 addr, + uint64 page_size, + uint8 expected) +{ + clockcache *clock = (clockcache *)cc; + buffer_handle buffer; + platform_status rc = platform_buffer_init(&buffer, page_size); + if (!SUCCESS(rc)) { + return rc; + } + + rc = io_read(clock->io, platform_buffer_getaddr(&buffer), page_size, addr); + if (SUCCESS(rc)) { + const uint8 *bytes = platform_buffer_getaddr(&buffer); + for (uint64 i = 0; i < page_size; i++) { + if (bytes[i] != expected) { + platform_error_log("cache_test: unexpected byte at addr=%lu " + "offset=%lu: got=%u expected=%u\n", + addr, + i, + bytes[i], + expected); + rc = STATUS_TEST_FAILED; + break; + } + } + } + + platform_status deinit_rc = platform_buffer_deinit(&buffer); + if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { + rc = deinit_rc; + } + return rc; +} + +/* + * Corrupt one full raw-I/O page and make that corruption durable. The + * allocator clean-state records are intentionally outside the cache, so this + * lets the bootstrap test exercise A/B fallback without test-only allocator + * hooks. + */ +static platform_status +cache_test_zero_disk_page(io_handle *io, uint64 addr, uint64 page_size) +{ + buffer_handle buffer; + platform_status rc = platform_buffer_init(&buffer, page_size); + if (!SUCCESS(rc)) { + return rc; + } + + void *page = platform_buffer_getaddr(&buffer); + memset(page, 0, page_size); + rc = io_write(io, page, page_size, addr); + if (SUCCESS(rc)) { + rc = io_durable_barrier(io); + } + + platform_status deinit_rc = platform_buffer_deinit(&buffer); + if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { + rc = deinit_rc; + } + return rc; +} + +/* + * The recovery allocator must bootstrap only the extents it owns itself, then + * let recovery walkers rebuild all other ownership. In particular, it must + * not trust a refcount table persisted by an earlier clean shutdown. + */ +static platform_status +test_rc_allocator_recovery_bootstrap(allocator_config *cfg, + io_handle *io, + platform_heap_id hid) +{ + const allocator_root_id root_id = 1; + uint64 refcount_buffer_size = + ROUNDUP(cfg->extent_capacity * sizeof(refcount), cfg->io_cfg->page_size); + uint64 refcount_extent_count = + (refcount_buffer_size + cfg->io_cfg->extent_size - 1) + / cfg->io_cfg->extent_size; + uint64 reserved_extent_count = 1 + refcount_extent_count + 2; + uint64 old_clean_state_addr = + (1 + refcount_extent_count + 1) * cfg->io_cfg->extent_size; + uint64 super_addr = 0; + uint64 stale_extent_addr = 0; + platform_status rc = STATUS_OK; + rc_allocator original, recovery, remounted; + bool32 original_live = FALSE; + bool32 recovery_live = FALSE; + bool32 remounted_live = FALSE; + + ZERO_CONTENTS(&original); + ZERO_CONTENTS(&recovery); + ZERO_CONTENTS(&remounted); + platform_default_log("cache_test: allocator recovery bootstrap test started\n"); + + rc = rc_allocator_init( + &original, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + original_live = TRUE; + + rc = allocator_alloc_super_addr( + (allocator *)&original, root_id, &super_addr); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = allocator_alloc( + (allocator *)&original, &stale_extent_addr, PAGE_TYPE_MISC); + if (!SUCCESS(rc)) { + goto cleanup; + } + + /* Persist a non-reserved extent that the recovery mount must ignore. */ + rc_allocator_unmount(&original); + original_live = FALSE; + + /* + * New allocator initialization writes false records with sequences 1 and + * 2. The first clean close writes the true record into slot 0 (sequence + * 3), making slot 1 the older false record. Destroying that older slot + * must not prevent a normal mount from trusting the surviving true slot. + */ + rc = cache_test_zero_disk_page( + io, old_clean_state_addr, cfg->io_cfg->page_size); + if (!SUCCESS(rc)) { + goto cleanup; + } + + rc = rc_allocator_mount( + &remounted, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + platform_error_log("cache_test: normal mount did not fall back to its " + "surviving clean-state record\n"); + goto cleanup; + } + remounted_live = TRUE; + if (allocator_get_refcount((allocator *)&remounted, stale_extent_addr) + != AL_ONE_REF) + { + platform_error_log("cache_test: normal mount did not load the clean " + "refcount map\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + /* Simulate a crash after normal mount durably marked the allocator dirty. */ + rc_allocator_deinit(&remounted); + remounted_live = FALSE; + + rc = rc_allocator_mount( + &remounted, cfg, io, hid, platform_get_module_id()); + if (rc.r != STATUS_INVALID_STATE.r) { + platform_error_log("cache_test: normal mount trusted an allocator after " + "a simulated mounted crash\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc = STATUS_OK; + + rc = rc_allocator_mount_recovery( + &recovery, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + recovery_live = TRUE; + + uint64 mounted_super_addr = 0; + rc = allocator_get_super_addr( + (allocator *)&recovery, root_id, &mounted_super_addr); + if (!SUCCESS(rc) || mounted_super_addr != super_addr) { + platform_error_log("cache_test: recovery lost a persisted superblock " + "mapping\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + if (allocator_in_use((allocator *)&recovery) != reserved_extent_count) { + platform_error_log("cache_test: recovery expected %lu reserved extents, " + "found %lu\n", + reserved_extent_count, + allocator_in_use((allocator *)&recovery)); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + for (uint64 extent_no = 0; extent_no < reserved_extent_count; extent_no++) + { + uint64 extent_addr = extent_no * cfg->io_cfg->extent_size; + if (allocator_get_refcount((allocator *)&recovery, extent_addr) + != AL_ONE_REF) + { + platform_error_log("cache_test: recovery did not reserve extent %lu\n", + extent_no); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + } + if (allocator_get_refcount((allocator *)&recovery, stale_extent_addr) + != AL_FREE) + { + platform_error_log("cache_test: recovery reused a persisted data " + "refcount\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + rc = rc_allocator_rebuild_acquire_extent(&recovery, stale_extent_addr); + if (!SUCCESS(rc) + || allocator_get_refcount((allocator *)&recovery, stale_extent_addr) + != AL_ONE_REF) + { + platform_error_log("cache_test: first rebuilt extent reference was " + "incorrect\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + /* + * Abort never persists a partial rebuild and leaves the allocator marked + * unclean, so a normal mount must refuse to trust the old refcount map. + */ + rc_allocator_abort_recovery(&recovery); + recovery_live = FALSE; + + rc = rc_allocator_mount( + &remounted, cfg, io, hid, platform_get_module_id()); + if (rc.r != STATUS_INVALID_STATE.r) { + platform_error_log("cache_test: normal mount trusted an unclean " + "allocator map\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc = STATUS_OK; + + rc = rc_allocator_mount_recovery( + &recovery, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + recovery_live = TRUE; + + rc = rc_allocator_rebuild_acquire_extent(&recovery, stale_extent_addr); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = rc_allocator_rebuild_acquire_extent(&recovery, stale_extent_addr); + if (!SUCCESS(rc) + || allocator_get_refcount((allocator *)&recovery, stale_extent_addr) + != AL_ONE_REF + 1) + { + platform_error_log("cache_test: rebuilt extent reference did not " + "increment\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc_allocator_rebuild_finish(&recovery); + rc_allocator_unmount(&recovery); + recovery_live = FALSE; + + rc = rc_allocator_mount( + &remounted, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + remounted_live = TRUE; + if (allocator_get_refcount((allocator *)&remounted, stale_extent_addr) + != AL_ONE_REF + 1) + { + platform_error_log("cache_test: finished recovery was not persisted by " + "clean unmount\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + +cleanup: + if (remounted_live) { + rc_allocator_deinit(&remounted); + } + if (recovery_live) { + if (recovery.recovery_in_progress) { + rc_allocator_abort_recovery(&recovery); + } else { + rc_allocator_deinit(&recovery); + } + } + if (original_live) { + rc_allocator_deinit(&original); + } + + if (SUCCESS(rc)) { + platform_default_log("cache_test: allocator recovery bootstrap test " + "passed\n"); + } + return rc; +} + +typedef struct { + cache *cc; + volatile bool32 finished; + platform_status status; +} cache_fence_test_context; + +/* Mirrors clockcache's private CC_WRITEBACK_REQUESTED status bit. */ +#define CACHE_TEST_WRITEBACK_REQUESTED (1u << 7) + +static void +cache_test_writeback_fence_thread(void *arg) +{ + cache_fence_test_context *ctxt = (cache_fence_test_context *)arg; + ctxt->status = cache_writeback_fence(ctxt->cc); + __atomic_store_n(&ctxt->finished, TRUE, __ATOMIC_RELEASE); +} + +/* + * Verify that a fence drains an already-dirty page without waiting for a page + * dirtied after its cut. Keeping old_page write-locked forces the fence to + * remain in progress long enough to deterministically dirty new_page after + * the dirty-generation rotation. + */ +static platform_status +test_cache_writeback_fence(cache *cc, + clockcache_config *cfg, + platform_heap_id hid) +{ + platform_status rc = STATUS_OK; + uint64 pages_per_extent = + cache_config_pages_per_extent(&cfg->super); + uint64 *addr_arr = NULL; + page_handle *old_page = NULL; + page_handle *new_page = NULL; + bool32 old_claimed = FALSE, old_locked = FALSE; + bool32 new_claimed = FALSE, new_locked = FALSE; + bool32 fence_thread_started = FALSE; + const uint8 old_baseline = 0x11, old_value = 0x22; + const uint8 new_baseline = 0x33, new_value = 0x44; + cache_fence_test_context fence_ctxt = { + .cc = cc, .finished = FALSE, .status = STATUS_OK}; + platform_thread fence_thread; + + platform_default_log("cache_test: writeback fence test started\n"); + platform_assert(cfg->page_capacity >= 2 * pages_per_extent); + + addr_arr = TYPED_ARRAY_MALLOC(hid, addr_arr, 2 * pages_per_extent); + if (addr_arr == NULL) { + rc = STATUS_NO_MEMORY; + goto cleanup; + } + + rc = cache_test_alloc_extents(cc, cfg, addr_arr, 2); + if (!SUCCESS(rc)) { + platform_free(hid, addr_arr); + addr_arr = NULL; + goto cleanup; + } + + rc = cache_test_fill_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), old_baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = cache_test_fill_page(cc, + addr_arr[pages_per_extent], + cache_config_page_size(&cfg->super), + new_baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + cache_flush(cc); + + old_page = cache_get(cc, addr_arr[0], TRUE, PAGE_TYPE_MISC); + if (!cache_try_claim(cc, old_page)) { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + old_claimed = TRUE; + cache_lock(cc, old_page); + old_locked = TRUE; + memset(old_page->data, + old_value, + cache_config_page_size(&cfg->super)); + cache_mark_dirty(cc, old_page); + + clockcache *clock = (clockcache *)cc; + uint64 initial_generation = + __atomic_load_n(&clock->dirty_generation, __ATOMIC_ACQUIRE); + rc = platform_thread_create(&fence_thread, + FALSE, + cache_test_writeback_fence_thread, + &fence_ctxt, + hid); + if (!SUCCESS(rc)) { + goto cleanup; + } + fence_thread_started = TRUE; + + timestamp wait_start = platform_get_timestamp(); + while (__atomic_load_n(&clock->dirty_generation, __ATOMIC_ACQUIRE) + == initial_generation) + { + if (platform_timestamp_elapsed(wait_start) > SEC_TO_NSEC(5)) { + platform_error_log("cache_test: writeback fence did not take a cut\n"); + rc = STATUS_TIMEDOUT; + goto cleanup; + } + platform_sleep_ns(1000); + } + + new_page = cache_get( + cc, addr_arr[pages_per_extent], TRUE, PAGE_TYPE_MISC); + if (!cache_try_claim(cc, new_page)) { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + new_claimed = TRUE; + cache_lock(cc, new_page); + new_locked = TRUE; + memset(new_page->data, + new_value, + cache_config_page_size(&cfg->super)); + cache_mark_dirty(cc, new_page); + cache_unlock(cc, new_page); + new_locked = FALSE; + cache_unclaim(cc, new_page); + new_claimed = FALSE; + cache_unget(cc, new_page); + new_page = NULL; + + if (__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { + platform_error_log("cache_test: fence completed while old page was locked\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + +cleanup: + if (new_locked) { + cache_unlock(cc, new_page); + } + if (new_claimed) { + cache_unclaim(cc, new_page); + } + if (new_page != NULL) { + cache_unget(cc, new_page); + } + if (old_locked) { + cache_unlock(cc, old_page); + } + if (old_claimed) { + cache_unclaim(cc, old_page); + } + if (old_page != NULL) { + cache_unget(cc, old_page); + } + if (fence_thread_started) { + platform_status join_rc = platform_thread_join(&fence_thread); + if (!SUCCESS(join_rc) && SUCCESS(rc)) { + rc = join_rc; + } + if (!SUCCESS(fence_ctxt.status) && SUCCESS(rc)) { + rc = fence_ctxt.status; + } + } + + if (SUCCESS(rc)) { + uint32 dirty_count = cache_count_dirty(cc); + if (dirty_count != 1) { + platform_error_log("cache_test: expected one post-cut dirty page, " + "found %u\n", + dirty_count); + rc = STATUS_TEST_FAILED; + } + } + if (SUCCESS(rc)) { + rc = cache_durable_barrier(cc); + } + if (SUCCESS(rc)) { + rc = cache_test_verify_disk_page(cc, + addr_arr[0], + cache_config_page_size(&cfg->super), + old_value); + } + if (SUCCESS(rc)) { + /* The post-cut page remains dirty, so its baseline must still be disk. */ + rc = cache_test_verify_disk_page(cc, + addr_arr[pages_per_extent], + cache_config_page_size(&cfg->super), + new_baseline); + } + + if (addr_arr != NULL) { + cache_flush(cc); + for (uint32 i = 0; i < 2; i++) { + uint64 addr = addr_arr[i * pages_per_extent]; + allocator *al = cache_get_allocator(cc); + refcount ref = allocator_dec_ref(al, addr, PAGE_TYPE_MISC); + platform_assert(ref == AL_NO_REFS); + cache_extent_discard(cc, addr, PAGE_TYPE_MISC); + ref = allocator_dec_ref(al, addr, PAGE_TYPE_MISC); + platform_assert(ref == AL_FREE); + } + platform_free(hid, addr_arr); + } + + if (SUCCESS(rc)) { + platform_default_log("cache_test: writeback fence test passed\n"); + } else { + platform_default_log("cache_test: writeback fence test failed\n"); + } + return rc; +} + +/* + * A fence must allow an already-claimed writer to finish its selected dirty + * interval, but it must not complete before that writer releases the claim. + */ +static platform_status +test_cache_writeback_fence_claimed_page(cache *cc, + clockcache_config *cfg, + platform_heap_id hid) +{ + platform_status rc = STATUS_OK; + uint64 pages_per_extent = + cache_config_pages_per_extent(&cfg->super); + uint64 *addr_arr = NULL; + page_handle *page = NULL; + bool32 claimed = FALSE, locked = FALSE; + bool32 fence_thread_started = FALSE; + const uint8 baseline = 0x55, final_value = 0x66; + cache_fence_test_context fence_ctxt = { + .cc = cc, .finished = FALSE, .status = STATUS_OK}; + platform_thread fence_thread; + + platform_default_log("cache_test: claimed-page fence test started\n"); + + addr_arr = TYPED_ARRAY_MALLOC(hid, addr_arr, pages_per_extent); + if (addr_arr == NULL) { + rc = STATUS_NO_MEMORY; + goto cleanup; + } + + rc = cache_test_alloc_extents(cc, cfg, addr_arr, 1); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = cache_test_fill_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + cache_flush(cc); + + page = cache_get(cc, addr_arr[0], TRUE, PAGE_TYPE_MISC); + if (!cache_try_claim(cc, page)) { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + claimed = TRUE; + cache_lock(cc, page); + locked = TRUE; + memset(page->data, final_value, cache_config_page_size(&cfg->super)); + cache_unlock(cc, page); + locked = FALSE; + + clockcache *clock = (clockcache *)cc; + uint64 initial_generation = + __atomic_load_n(&clock->dirty_generation, __ATOMIC_ACQUIRE); + rc = platform_thread_create(&fence_thread, + FALSE, + cache_test_writeback_fence_thread, + &fence_ctxt, + hid); + if (!SUCCESS(rc)) { + goto cleanup; + } + fence_thread_started = TRUE; + + uint64 entry_no = + clock->lookup[addr_arr[0] >> clock->cfg->log_page_size]; + platform_assert(entry_no < clock->cfg->page_capacity); + timestamp wait_start = platform_get_timestamp(); + while ((__atomic_load_n(&clock->dirty_generation, __ATOMIC_ACQUIRE) + == initial_generation) + || !(__atomic_load_n(&clock->entry[entry_no].status, + __ATOMIC_ACQUIRE) + & CACHE_TEST_WRITEBACK_REQUESTED)) + { + if (platform_timestamp_elapsed(wait_start) > SEC_TO_NSEC(5)) { + platform_error_log("cache_test: fence did not request claimed page\n"); + rc = STATUS_TIMEDOUT; + goto cleanup; + } + platform_sleep_ns(1000); + } + + if (__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { + platform_error_log("cache_test: fence completed while page was claimed\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + /* This claim predates the request, so it is permitted to finish. */ + cache_lock(cc, page); + locked = TRUE; + memset(page->data, final_value, cache_config_page_size(&cfg->super)); + cache_unlock(cc, page); + locked = FALSE; + cache_unclaim(cc, page); + claimed = FALSE; + cache_unget(cc, page); + page = NULL; + +cleanup: + if (locked) { + cache_unlock(cc, page); + } + if (claimed) { + cache_unclaim(cc, page); + } + if (page != NULL) { + cache_unget(cc, page); + } + if (fence_thread_started) { + platform_status join_rc = platform_thread_join(&fence_thread); + if (!SUCCESS(join_rc) && SUCCESS(rc)) { + rc = join_rc; + } + if (!SUCCESS(fence_ctxt.status) && SUCCESS(rc)) { + rc = fence_ctxt.status; + } + } + + if (SUCCESS(rc) && cache_count_dirty(cc) != 0) { + platform_error_log("cache_test: claimed-page fence left dirty pages\n"); + rc = STATUS_TEST_FAILED; + } + if (SUCCESS(rc)) { + rc = cache_durable_barrier(cc); + } + if (SUCCESS(rc)) { + rc = cache_test_verify_disk_page(cc, + addr_arr[0], + cache_config_page_size(&cfg->super), + final_value); + } + + if (addr_arr != NULL) { + cache_flush(cc); + allocator *al = cache_get_allocator(cc); + refcount ref = allocator_dec_ref(al, addr_arr[0], PAGE_TYPE_MISC); + platform_assert(ref == AL_NO_REFS); + cache_extent_discard(cc, addr_arr[0], PAGE_TYPE_MISC); + ref = allocator_dec_ref(al, addr_arr[0], PAGE_TYPE_MISC); + platform_assert(ref == AL_FREE); + platform_free(hid, addr_arr); + } + + if (SUCCESS(rc)) { + platform_default_log("cache_test: claimed-page fence test passed\n"); + } else { + platform_default_log("cache_test: claimed-page fence test failed\n"); + } + return rc; +} + platform_status test_cache_basic(cache *cc, clockcache_config *cfg, platform_heap_id hid) { @@ -98,6 +780,16 @@ test_cache_basic(cache *cc, clockcache_config *cfg, platform_heap_id hid) page_handle **page_arr = NULL; uint64 *addr_arr = NULL; + rc = test_cache_writeback_fence(cc, cfg, hid); + if (!SUCCESS(rc)) { + goto exit; + } + + rc = test_cache_writeback_fence_claimed_page(cc, cfg, hid); + if (!SUCCESS(rc)) { + goto exit; + } + /* allocate twice as many pages as the cache capacity */ uint64 pages_per_extent = cache_config_pages_per_extent(&cfg->super); uint32 extent_capacity = cfg->page_capacity / pages_per_extent; @@ -977,6 +1669,12 @@ cache_test(int argc, char *argv[]) goto cleanup; } + rc = test_rc_allocator_recovery_bootstrap( + &system_cfg.allocator_cfg, io, hid); + if (!SUCCESS(rc)) { + goto destroy_iohandle; + } + rc = test_init_task_system(&ts, hid, &system_cfg.task_cfg); if (!SUCCESS(rc)) { platform_error_log("Failed to init splinter state: %s\n", diff --git a/tests/functional/log_test.c b/tests/functional/log_test.c index 1bf96ba5..6a16d05f 100644 --- a/tests/functional/log_test.c +++ b/tests/functional/log_test.c @@ -19,6 +19,8 @@ #include "poison.h" +#define LOG_TEST_LEAVES_PER_MEMTABLE 97 + int test_log_crash(clockcache *cc, clockcache_config *cache_cfg, @@ -59,12 +61,45 @@ test_log_crash(clockcache *cc, merge_accumulator_init(&msg, hid); for (i = 0; i < num_entries; i++) { + uint64 entry_num = i; + if (2 * LOG_TEST_LEAVES_PER_MEMTABLE <= num_entries + && i < 2 * LOG_TEST_LEAVES_PER_MEMTABLE) + { + /* + * Write the first two generations in the opposite order, and each + * generation in reverse leaf order. The iterator must restore the + * (memtable_generation, leaf_generation) order below. + */ + uint64 memtable_generation = 1 - i / LOG_TEST_LEAVES_PER_MEMTABLE; + uint64 leaf_generation = LOG_TEST_LEAVES_PER_MEMTABLE - 1 + - i % LOG_TEST_LEAVES_PER_MEMTABLE; + entry_num = memtable_generation * LOG_TEST_LEAVES_PER_MEMTABLE + + leaf_generation; + } key skey = - test_key(&keybuffer, TEST_RANDOM, i, 0, 0, 1 + (i % key_size), 0); - generate_test_message(gen, i, &msg); - log_write(logh, skey, merge_accumulator_to_message(&msg), i); + test_key(&keybuffer, + TEST_RANDOM, + entry_num, + 0, + 0, + 1 + (entry_num % key_size), + 0); + generate_test_message(gen, entry_num, &msg); + int log_rc = log_write(logh, + skey, + merge_accumulator_to_message(&msg), + entry_num / LOG_TEST_LEAVES_PER_MEMTABLE, + entry_num % LOG_TEST_LEAVES_PER_MEMTABLE); + platform_assert(log_rc == 0); } + rc = log_seal(logh); + platform_assert_status_ok(rc); + rc = cache_writeback_fence((cache *)cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier((cache *)cc); + platform_assert_status_ok(rc); + if (crash) { clockcache_deinit(cc); rc = clockcache_init( @@ -82,6 +117,13 @@ test_log_crash(clockcache *cc, generate_test_message(gen, i, &msg); message mmessage = merge_accumulator_to_message(&msg); iterator_curr(itorh, &returned_key, &returned_message); + uint64 memtable_generation; + uint64 leaf_generation; + shard_log_iterator_curr_generations( + &itor, &memtable_generation, &leaf_generation); + platform_assert( + memtable_generation == i / LOG_TEST_LEAVES_PER_MEMTABLE); + platform_assert(leaf_generation == i % LOG_TEST_LEAVES_PER_MEMTABLE); if (data_key_compare(cfg->data_cfg, skey, returned_key) || message_lex_cmp(mmessage, returned_message)) { @@ -99,6 +141,8 @@ test_log_crash(clockcache *cc, } platform_default_log("log returned %lu of %lu entries\n", i, num_entries); + platform_assert(i == num_entries); + platform_assert(!iterator_can_curr(itorh)); merge_accumulator_deinit(&msg); @@ -108,6 +152,182 @@ test_log_crash(clockcache *cc, return 0; } +static void +test_log_write_range(log_handle *logh, + test_message_generator *gen, + platform_heap_id hid, + uint64 key_size, + uint64 first_entry, + uint64 num_entries) +{ + merge_accumulator msg; + DECLARE_AUTO_KEY_BUFFER(keybuffer, hid); + merge_accumulator_init(&msg, hid); + + for (uint64 i = 0; i < num_entries; i++) { + uint64 entry_num = first_entry + i; + key skey = test_key(&keybuffer, + TEST_RANDOM, + entry_num, + 0, + 0, + 1 + (entry_num % key_size), + 0); + generate_test_message(gen, entry_num, &msg); + int log_rc = log_write(logh, + skey, + merge_accumulator_to_message(&msg), + entry_num, + 0); + platform_assert(log_rc == 0); + } + + merge_accumulator_deinit(&msg); +} + +static void +test_log_verify_segment(cache *cc, + shard_log_config *cfg, + const log_segment_info *segment, + test_message_generator *gen, + platform_heap_id hid, + uint64 key_size, + uint64 first_entry, + uint64 num_entries) +{ + shard_log_iterator itor; + iterator *itorh = (iterator *)&itor; + merge_accumulator msg; + DECLARE_AUTO_KEY_BUFFER(keybuffer, hid); + key returned_key; + message returned_message; + + platform_assert(segment->addr != 0); + platform_assert(segment->meta_addr != 0); + platform_status rc = shard_log_iterator_init( + cc, cfg, hid, segment->addr, segment->magic, &itor); + platform_assert_status_ok(rc); + + merge_accumulator_init(&msg, hid); + for (uint64 i = 0; i < num_entries; i++) { + uint64 entry_num = first_entry + i; + platform_assert(iterator_can_curr(itorh)); + key skey = test_key(&keybuffer, + TEST_RANDOM, + entry_num, + 0, + 0, + 1 + (entry_num % key_size), + 0); + generate_test_message(gen, entry_num, &msg); + iterator_curr(itorh, &returned_key, &returned_message); + uint64 memtable_generation; + uint64 leaf_generation; + shard_log_iterator_curr_generations( + &itor, &memtable_generation, &leaf_generation); + platform_assert(memtable_generation == entry_num); + platform_assert(leaf_generation == 0); + platform_assert(data_key_compare(cfg->data_cfg, skey, returned_key) == 0); + platform_assert(message_lex_cmp( + merge_accumulator_to_message(&msg), returned_message) + == 0); + rc = iterator_next(itorh); + platform_assert_status_ok(rc); + } + platform_assert(!iterator_can_curr(itorh)); + + merge_accumulator_deinit(&msg); + shard_log_iterator_deinit(hid, &itor); +} + +/* + * A rotation must permanently detach the first mini-allocator stream and + * produce a fresh one. Reinitializing the cache after each forced physical + * persistence cut makes this test exercise only persisted pages for both + * identities; it does not model logical durable-tail publication. + */ +static int +test_log_rotate(clockcache *cc, + clockcache_config *cache_cfg, + io_handle *io, + allocator *al, + shard_log_config *cfg, + shard_log *log, + platform_heap_id hid, + test_message_generator *gen, + uint64 key_size) +{ + const uint64 old_first = 1000, old_count = 16; + const uint64 new_first = 2000, new_count = 16; + log_segment_info sealed, fresh; + + platform_status rc = shard_log_init(log, (cache *)cc, cfg); + platform_assert_status_ok(rc); + + test_log_write_range( + (log_handle *)log, gen, hid, key_size, old_first, old_count); + rc = log_rotate((log_handle *)log, &sealed, &fresh); + platform_assert_status_ok(rc); + platform_assert(sealed.addr != 0); + platform_assert(sealed.meta_addr != 0); + platform_assert(fresh.addr != 0); + platform_assert(fresh.meta_addr != 0); + platform_assert(sealed.meta_addr != fresh.meta_addr); + platform_assert(sealed.magic != fresh.magic); + + rc = cache_writeback_fence((cache *)cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier((cache *)cc); + platform_assert_status_ok(rc); + + clockcache_deinit(cc); + rc = clockcache_init( + cc, cache_cfg, io, al, "rotated-old", hid, platform_get_module_id()); + platform_assert_status_ok(rc); + test_log_verify_segment((cache *)cc, + cfg, + &sealed, + gen, + hid, + key_size, + old_first, + old_count); + + test_log_write_range( + (log_handle *)log, gen, hid, key_size, new_first, new_count); + rc = log_seal((log_handle *)log); + platform_assert_status_ok(rc); + rc = cache_writeback_fence((cache *)cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier((cache *)cc); + platform_assert_status_ok(rc); + + clockcache_deinit(cc); + rc = clockcache_init( + cc, cache_cfg, io, al, "rotated-new", hid, platform_get_module_id()); + platform_assert_status_ok(rc); + test_log_verify_segment((cache *)cc, + cfg, + &sealed, + gen, + hid, + key_size, + old_first, + old_count); + test_log_verify_segment((cache *)cc, + cfg, + &fresh, + gen, + hid, + key_size, + new_first, + new_count); + + shard_log_segment_discard((cache *)cc, &sealed); + shard_log_zap(log); + return 0; +} + static int test_log_large_message(cache *cc, shard_log_config *cfg, @@ -134,7 +354,8 @@ test_log_large_message(cache *cc, memset(merge_accumulator_data(&msg), 'L', value_len); int log_rc = - log_write((log_handle *)log, skey, merge_accumulator_to_message(&msg), 0); + log_write( + (log_handle *)log, skey, merge_accumulator_to_message(&msg), 0, 0); platform_assert(log_rc == 0); merge_accumulator filler; @@ -146,11 +367,22 @@ test_log_large_message(cache *cc, merge_accumulator_data(&filler), 'f', merge_accumulator_length(&filler)); for (uint64 i = 1; i < 16; i++) { log_rc = log_write( - (log_handle *)log, skey, merge_accumulator_to_message(&filler), i); + (log_handle *)log, + skey, + merge_accumulator_to_message(&filler), + i / 4, + i % 4); platform_assert(log_rc == 0); } merge_accumulator_deinit(&filler); + rc = log_seal((log_handle *)log); + platform_assert_status_ok(rc); + rc = cache_writeback_fence(cc); + platform_assert_status_ok(rc); + rc = cache_durable_barrier(cc); + platform_assert_status_ok(rc); + rc = shard_log_iterator_init(cc, cfg, hid, @@ -201,7 +433,12 @@ test_log_thread(void *arg) for (i = thread_id * num_entries; i < (thread_id + 1) * num_entries; i++) { key skey = test_key(&keybuf, TEST_RANDOM, i, 0, 0, key_size, 0); generate_test_message(gen, i, &msg); - log_write(logh, skey, merge_accumulator_to_message(&msg), i); + int log_rc = log_write(logh, + skey, + merge_accumulator_to_message(&msg), + i / 1024, + i % 1024); + platform_assert(log_rc == 0); } merge_accumulator_deinit(&msg); @@ -380,6 +617,17 @@ log_test(int argc, char *argv[]) rc = test_log_large_message((cache *)cc, &system_cfg.log_cfg, log, hid); platform_assert(rc == 0); + rc = test_log_rotate(cc, + &system_cfg.cache_cfg, + io, + (allocator *)&al, + &system_cfg.log_cfg, + log, + hid, + &gen, + workload_cfg.key_size); + platform_assert(rc == 0); + if (run_perf_test) { ret = test_log_perf((cache *)cc, &system_cfg.log_cfg, diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index 5377cde9..9c550ec0 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -250,6 +250,85 @@ CTEST2(splinter, test_inserts) core_destroy(&spl); } +/* + * The second checkpoint slot is a torn-write fallback, not permission for a + * normal mount to silently roll back past a newer, valid active record. A + * successful mount publishes such an active record; until crash recovery is + * implemented, a concurrent/restarted normal mount must reject it even + * though the preceding clean record remains valid in the other slot. + */ +CTEST2(splinter, test_mount_rejects_newer_active_checkpoint) +{ + allocator *alp = (allocator *)&data->al; + allocator_root_id root_id = test_generate_allocator_root_id(); + core_handle created, mounted, rejected, cleanup; + platform_status rc; + + rc = core_mkfs(&created, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + /* Give the clean record a real COW root, not just the empty-tree root. */ + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + merge_accumulator msg; + merge_accumulator_init(&msg, data->hid); + test_key(&keybuf, + TEST_RANDOM, + 1, + 0, + 0, + data->workload_cfg->key_size, + 0); + generate_test_message(&data->gen, 1, &msg); + rc = core_insert(&created, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); + merge_accumulator_deinit(&msg); + ASSERT_TRUE(SUCCESS(rc)); + + rc = core_unmount(&created); + ASSERT_TRUE(SUCCESS(rc)); + + /* This mount advances the A/B sequence with an unmounted=FALSE record. */ + rc = core_mount(&mounted, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + rc = core_mount(&rejected, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(STATUS_IS_EQ(rc, STATUS_INVALID_STATE)); + + /* Finish cleanly, then prove the same record pair is mountable again. */ + rc = core_unmount(&mounted); + ASSERT_TRUE(SUCCESS(rc)); + + rc = core_mount(&cleanup, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + &data->tasks, + root_id, + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + core_destroy(&cleanup); +} + static void trunk_shadow_init(trunk_shadow *shadow, data_config *data_cfg, From 4c6c6829432ac8192acd575c85ae64b2280bdf1c Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 17 Jul 2026 13:22:25 -0700 Subject: [PATCH 02/64] clarify durability semantics Signed-off-by: Rob Johnson --- src/blob.c | 2 +- src/cache.h | 55 ++++++++++++++----------- src/clockcache.c | 69 ++++++++++++++++++-------------- src/core.c | 4 +- src/platform_linux/platform_io.h | 8 +++- src/shard_log.c | 2 +- 6 files changed, 82 insertions(+), 58 deletions(-) diff --git a/src/blob.c b/src/blob.c index b6c8e084..9766c4dc 100644 --- a/src/blob.c +++ b/src/blob.c @@ -327,7 +327,7 @@ blob_sync(cache *cc, slice sblob) if (!SUCCESS(rc)) { break; } - cache_page_sync(cc, itor.page, FALSE, PAGE_TYPE_BLOB); + cache_page_writeback(cc, itor.page, FALSE, PAGE_TYPE_BLOB); blob_page_iterator_advance_page(&itor); } diff --git a/src/cache.h b/src/cache.h index 52bec143..a5c27e68 100644 --- a/src/cache.h +++ b/src/cache.h @@ -130,13 +130,13 @@ typedef async_status (*page_get_async_fn)(void *payload); typedef page_handle *(*page_get_async_state_result_fn)(void *payload); typedef bool32 (*page_try_claim_fn)(cache *cc, page_handle *page); -typedef void (*page_sync_fn)(cache *cc, - page_handle *page, - bool32 is_blocking, - page_type type); -typedef void (*extent_sync_fn)(cache *cc, - uint64 addr, - uint64 *pages_outstanding); +typedef void (*page_writeback_fn)(cache *cc, + page_handle *page, + bool32 is_blocking, + page_type type); +typedef void (*extent_writeback_fn)(cache *cc, + uint64 addr, + uint64 *pages_outstanding); typedef void (*page_prefetch_fn)(cache *cc, uint64 addr, page_type type); typedef int (*evict_fn)(cache *cc, bool32 ignore_pinned); typedef bool32 (*page_addr_pred_fn)(cache *cc, uint64 addr); @@ -175,8 +175,8 @@ typedef struct cache_ops { page_generic_fn page_mark_dirty; page_generic_fn page_pin; page_generic_fn page_unpin; - page_sync_fn page_sync; - extent_sync_fn extent_sync; + page_writeback_fn page_writeback; + extent_writeback_fn extent_writeback; cache_generic_fn flush; cache_writeback_fence_fn writeback_fence; cache_durable_barrier_fn durable_barrier; @@ -489,37 +489,44 @@ cache_unpin(cache *cc, page_handle *page) /* *----------------------------------------------------------------------------- - * cache_page_sync + * cache_page_writeback * - * Asynchronously writes the page back to disk. + * Issues writeback of the page to disk. This does NOT make the page durable; + * it only hands the write to the I/O layer (no device-cache flush). * - * This is used to sync log pages opportunistically. The current API doesn't - * inform the user when this happens. "It's not the ideal API." -- @aconway + * With is_blocking == FALSE the writeback is issued asynchronously and this + * returns without waiting for completion; there is no per-call way to observe + * when it finishes. With is_blocking == TRUE the page is written synchronously + * and the write has completed by the time this returns. *----------------------------------------------------------------------------- */ static inline void -cache_page_sync(cache *cc, - page_handle *page, - bool32 is_blocking, - page_type type) +cache_page_writeback(cache *cc, + page_handle *page, + bool32 is_blocking, + page_type type) { - return cc->ops->page_sync(cc, page, is_blocking, type); + return cc->ops->page_writeback(cc, page, is_blocking, type); } /* *----------------------------------------------------------------------------- - * cache_extent_sync + * cache_extent_writeback * - * Asynchronously syncs the extent beginning at addr. + * Asynchronously issues writeback of the extent beginning at addr. This does + * NOT make the extent durable; it only hands the writes to the I/O layer (no + * device-cache flush) and returns without waiting for completion. * * *pages_outstanding is immediately incremented by the number of pages * issued for writeback (the non-clean pages of the extent); as writebacks - * complete, *pages_outstanding is decremented atomically. + * complete, *pages_outstanding is decremented atomically. This counter is the + * only way to observe when the issued writebacks finish. * * Assumes pages_outstanding is an aligned uint64, so (on x86) the caller * can access it and observe its value atomically. * - * TODO: What happens if two callers call cache_extent_sync on the same extent? + * TODO: What happens if two callers call cache_extent_writeback on the same + * extent? * * All pages in the extent must be clean or cleanable. * The page may not be in writeback, loading, or locked, or claimed, otherwise @@ -527,9 +534,9 @@ cache_page_sync(cache *cc, *----------------------------------------------------------------------------- */ static inline void -cache_extent_sync(cache *cc, uint64 addr, uint64 *pages_outstanding) +cache_extent_writeback(cache *cc, uint64 addr, uint64 *pages_outstanding) { - cc->ops->extent_sync(cc, addr, pages_outstanding); + cc->ops->extent_writeback(cc, addr, pages_outstanding); } /* diff --git a/src/clockcache.c b/src/clockcache.c index 74aba087..760a2a70 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -2540,17 +2540,22 @@ clockcache_unpin(clockcache *cc, page_handle *page) /* *----------------------------------------------------------------------------- - * clockcache_page_sync -- + * clockcache_page_writeback -- * - * Asynchronously syncs the page. Currently there is no way to check - *when the writeback has completed. + * Issues writeback of the page. This does not make the page durable; it + * only hands the write to the I/O layer. + * + * With is_blocking == FALSE the writeback is issued asynchronously and + * this returns without waiting for completion. With is_blocking == TRUE + * the page is written synchronously and has completed by the time this + * returns. *----------------------------------------------------------------------------- */ void -clockcache_page_sync(clockcache *cc, - page_handle *page, - bool32 is_blocking, - page_type type) +clockcache_page_writeback(clockcache *cc, + page_handle *page, + bool32 is_blocking, + page_type type) { uint32 entry_number = clockcache_page_to_entry_number(cc, page); async_io_state *state; @@ -2574,7 +2579,7 @@ clockcache_page_sync(clockcache *cc, } platform_assert(0, - "page_sync requires a cleanable page: entry=%u " + "page_writeback requires a cleanable page: entry=%u " "status=%u\n", entry_number, clockcache_get_status(cc, entry_number)); @@ -2588,7 +2593,7 @@ clockcache_page_sync(clockcache *cc, if (!is_blocking) { state = TYPED_MALLOC(PROCESS_PRIVATE_HEAP_ID, state); if (state == NULL) { - platform_error_log("clockcache_page_sync: async_io_state allocation " + platform_error_log("clockcache_page_writeback: async_io_state allocation " "failed for addr %lu, entry %u, type %u\n", addr, entry_number, @@ -2604,7 +2609,7 @@ clockcache_page_sync(clockcache *cc, clockcache_write_callback, state); if (!SUCCESS(status)) { - platform_error_log("clockcache_page_sync: io_async_state_init failed " + platform_error_log("clockcache_page_writeback: io_async_state_init failed " "for addr %lu, entry %u, type %u: %s\n", addr, entry_number, @@ -2614,7 +2619,7 @@ clockcache_page_sync(clockcache *cc, platform_assert_status_ok(status); status = io_async_state_append_page(state->iostate, page->data); if (!SUCCESS(status)) { - platform_error_log("clockcache_page_sync: io_async_state_append_page " + platform_error_log("clockcache_page_writeback: io_async_state_append_page " "failed for addr %lu, entry %u, type %u: %s\n", addr, entry_number, @@ -2626,7 +2631,7 @@ clockcache_page_sync(clockcache *cc, } else { status = io_write(cc->io, page->data, clockcache_page_size(cc), addr); if (!SUCCESS(status)) { - platform_error_log("clockcache_page_sync: io_write failed for addr " + platform_error_log("clockcache_page_writeback: io_write failed for addr " "%lu, entry %u, type %u: %s\n", addr, entry_number, @@ -2636,7 +2641,7 @@ clockcache_page_sync(clockcache *cc, platform_assert_status_ok(status); clockcache_log(addr, entry_number, - "page_sync write entry %u addr %lu\n", + "page_writeback write entry %u addr %lu\n", entry_number, addr); clockcache_dirty_complete_writeback(cc, entry_number); @@ -2645,20 +2650,24 @@ clockcache_page_sync(clockcache *cc, /* *----------------------------------------------------------------------------- - * clockcache_extent_sync -- + * clockcache_extent_writeback -- * - * Asynchronously syncs the extent. + * Asynchronously issues writeback of the extent. This does not make the + * extent durable; it only hands the writes to the I/O layer and returns + * without waiting for completion. * * Adds the number of pages issued writeback to the counter pointed to * by pages_outstanding. When the writes complete, a callback subtracts * them off, so that the caller may track how many pages are in - *writeback. + * writeback. * * Assumes all pages in the extent are clean or cleanable *----------------------------------------------------------------------------- */ void -clockcache_extent_sync(clockcache *cc, uint64 addr, uint64 *pages_outstanding) +clockcache_extent_writeback(clockcache *cc, + uint64 addr, + uint64 *pages_outstanding) { async_io_state *state = NULL; uint64 i; @@ -2677,7 +2686,7 @@ clockcache_extent_sync(clockcache *cc, uint64 addr, uint64 *pages_outstanding) req_addr = page_addr; state = TYPED_MALLOC(PROCESS_PRIVATE_HEAP_ID, state); if (state == NULL) { - platform_error_log("clockcache_extent_sync: async_io_state " + platform_error_log("clockcache_extent_writeback: async_io_state " "allocation failed for extent addr %lu, " "page addr %lu, entry %u\n", addr, @@ -2694,7 +2703,7 @@ clockcache_extent_sync(clockcache *cc, uint64 addr, uint64 *pages_outstanding) clockcache_write_callback, state); if (!SUCCESS(rc)) { - platform_error_log("clockcache_extent_sync: " + platform_error_log("clockcache_extent_writeback: " "io_async_state_init failed for extent addr " "%lu, req addr %lu, entry %u: %s\n", addr, @@ -2707,7 +2716,7 @@ clockcache_extent_sync(clockcache *cc, uint64 addr, uint64 *pages_outstanding) platform_status rc = io_async_state_append_page( state->iostate, clockcache_get_entry(cc, entry_number)->page.data); if (!SUCCESS(rc)) { - platform_error_log("clockcache_extent_sync: " + platform_error_log("clockcache_extent_writeback: " "io_async_state_append_page failed for extent " "addr %lu, page addr %lu, entry %u: %s\n", addr, @@ -3453,20 +3462,22 @@ clockcache_get_async_state_result_virtual(void *payload) } void -clockcache_page_sync_virtual(cache *c, - page_handle *page, - bool32 is_blocking, - page_type type) +clockcache_page_writeback_virtual(cache *c, + page_handle *page, + bool32 is_blocking, + page_type type) { clockcache *cc = (clockcache *)c; - clockcache_page_sync(cc, page, is_blocking, type); + clockcache_page_writeback(cc, page, is_blocking, type); } void -clockcache_extent_sync_virtual(cache *c, uint64 addr, uint64 *pages_outstanding) +clockcache_extent_writeback_virtual(cache *c, + uint64 addr, + uint64 *pages_outstanding) { clockcache *cc = (clockcache *)c; - clockcache_extent_sync(cc, addr, pages_outstanding); + clockcache_extent_writeback(cc, addr, pages_outstanding); } void @@ -3621,8 +3632,8 @@ static cache_ops clockcache_ops = { .page_mark_dirty = clockcache_mark_dirty_virtual, .page_pin = clockcache_pin_virtual, .page_unpin = clockcache_unpin_virtual, - .page_sync = clockcache_page_sync_virtual, - .extent_sync = clockcache_extent_sync_virtual, + .page_writeback = clockcache_page_writeback_virtual, + .extent_writeback = clockcache_extent_writeback_virtual, .flush = clockcache_flush_virtual, .writeback_fence = clockcache_writeback_fence_virtual, .durable_barrier = clockcache_durable_barrier_virtual, diff --git a/src/core.c b/src/core.c index 3faac28e..c8658799 100644 --- a/src/core.c +++ b/src/core.c @@ -277,7 +277,7 @@ core_write_checkpoint_page(core_handle *spl, cache_mark_dirty(spl->cc, page); cache_unlock(spl->cc, page); cache_unclaim(spl->cc, page); - cache_page_sync(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); + cache_page_writeback(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); cache_unget(spl->cc, page); } @@ -290,7 +290,7 @@ core_initialize_checkpoint_record_page(core_handle *spl, uint64 page_addr) cache_mark_dirty(spl->cc, page); cache_unlock(spl->cc, page); cache_unclaim(spl->cc, page); - cache_page_sync(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); + cache_page_writeback(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); cache_unget(spl->cc, page); } diff --git a/src/platform_linux/platform_io.h b/src/platform_linux/platform_io.h index 3ecd12e0..cb60e861 100644 --- a/src/platform_linux/platform_io.h +++ b/src/platform_linux/platform_io.h @@ -199,7 +199,13 @@ io_cleanup(io_handle *io, uint64 count) return io->ops->cleanup(io, count); } -// Guarantees all in-flight IOs are complete before return +// Wait until we have seen each process's I/O be quiescent. +// +// This means that all I/Os that were in-flight at the start of this call are +// guaranteed to have finished before this call returns. +// +// But I/Os that are issued after this call starts may not be complete when +// this call returns. static inline void io_wait_all(io_handle *io) { diff --git a/src/shard_log.c b/src/shard_log.c index 9c1fd016..cccf8afe 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -344,7 +344,7 @@ shard_log_write(log_handle *logh, cache_unlock(cc, page); cache_unclaim(cc, page); - cache_page_sync(cc, page, FALSE, PAGE_TYPE_LOG); + cache_page_writeback(cc, page, FALSE, PAGE_TYPE_LOG); cache_unget(cc, page); if (get_new_page_for_thread(log, thread_data, &page)) { From 6954ceb01375b46b7dc94e7204aef810cd2cb97d Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 17 Jul 2026 13:51:50 -0700 Subject: [PATCH 03/64] get rid of mark_dirty Signed-off-by: Rob Johnson --- src/blob.c | 1 - src/btree.c | 3 -- src/cache.h | 92 +++++++++++++---------------------- src/clockcache.c | 29 ----------- src/core.c | 2 - src/mini_allocator.c | 1 - src/shard_log.c | 1 - src/trunk.c | 2 - tests/functional/cache_test.c | 4 -- 9 files changed, 34 insertions(+), 101 deletions(-) diff --git a/src/blob.c b/src/blob.c index 9766c4dc..f8c0c375 100644 --- a/src/blob.c +++ b/src/blob.c @@ -226,7 +226,6 @@ blob_page_iterator_get_curr(blob_page_iterator *iter, iter->cc, iter->fragment.addr, TRUE, PAGE_TYPE_BLOB); } cache_lock(iter->cc, iter->page); - cache_mark_dirty(iter->cc, iter->page); } } } diff --git a/src/btree.c b/src/btree.c index e93b0c50..9504aba3 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1197,7 +1197,6 @@ btree_node_lock(cache *cc, // IN btree_node *node) // IN { cache_lock(cc, node->page); - cache_mark_dirty(cc, node->page); } static inline void @@ -1294,8 +1293,6 @@ btree_create(cache *cc, btree_init_hdr(cfg, root.hdr); - cache_mark_dirty(cc, root.page); - // If this btree is for a memtable then pin all pages belonging to it if (pinned) { cache_pin(cc, root.page); diff --git a/src/cache.h b/src/cache.h index a5c27e68..1cecd131 100644 --- a/src/cache.h +++ b/src/cache.h @@ -165,37 +165,36 @@ typedef struct cache_ops { page_get_async_fn page_get_async; page_get_async_state_result_fn page_get_async_result; - page_generic_fn page_unget; - page_try_claim_fn page_try_claim; - page_generic_fn page_unclaim; - page_generic_fn page_lock; - page_generic_fn page_unlock; - page_prefetch_fn page_prefetch; - page_prefetch_fn page_prefetch_page; - page_generic_fn page_mark_dirty; - page_generic_fn page_pin; - page_generic_fn page_unpin; - page_writeback_fn page_writeback; - extent_writeback_fn extent_writeback; - cache_generic_fn flush; + page_generic_fn page_unget; + page_try_claim_fn page_try_claim; + page_generic_fn page_unclaim; + page_generic_fn page_lock; + page_generic_fn page_unlock; + page_prefetch_fn page_prefetch; + page_prefetch_fn page_prefetch_page; + page_generic_fn page_pin; + page_generic_fn page_unpin; + page_writeback_fn page_writeback; + extent_writeback_fn extent_writeback; + cache_generic_fn flush; cache_writeback_fence_fn writeback_fence; cache_durable_barrier_fn durable_barrier; - evict_fn evict; - cache_generic_fn cleanup; - page_addr_pred_fn in_use; - page_addr_fn assert_ungot; - cache_generic_fn assert_free; - validate_page_fn validate_page; - cache_present_fn cache_present; - cache_print_fn print; - cache_print_fn print_stats; - io_stats_fn io_stats; - cache_generic_fn reset_stats; - count_dirty_fn count_dirty; - page_get_read_ref_fn page_get_read_ref; - enable_sync_get_fn enable_sync_get; - get_allocator_fn get_allocator; - cache_config_fn get_config; + evict_fn evict; + cache_generic_fn cleanup; + page_addr_pred_fn in_use; + page_addr_fn assert_ungot; + cache_generic_fn assert_free; + validate_page_fn validate_page; + cache_present_fn cache_present; + cache_print_fn print; + cache_print_fn print_stats; + io_stats_fn io_stats; + cache_generic_fn reset_stats; + count_dirty_fn count_dirty; + page_get_read_ref_fn page_get_read_ref; + enable_sync_get_fn enable_sync_get; + get_allocator_fn get_allocator; + cache_config_fn get_config; } cache_ops; // To sub-class cache, make a cache your first field; @@ -370,10 +369,11 @@ cache_unclaim(cache *cc, page_handle *page) * * Blocks until outstanding read locks are released by other threads. * - * Clockcache conservatively begins a dirty interval on this transition, so a - * checkpoint fence cannot miss a caller that makes its first change before - * calling cache_mark_dirty(). cache_mark_dirty() remains the explicit, - * idempotent declaration of intent to modify the page. + * Acquiring the write lock marks the page dirty: it begins a dirty interval on + * this transition (before the caller's first change), so a checkpoint fence + * cannot miss the modification. A page obtained write-locked from cache_alloc() + * is likewise already dirty. Callers therefore do not separately declare the + * mutation. *---------------------------------------------------------------------- */ static inline void @@ -433,26 +433,6 @@ cache_prefetch_page(cache *cc, uint64 addr, page_type type) return cc->ops->page_prefetch_page(cc, addr, type); } -/* - *---------------------------------------------------------------------- - * cache_mark_dirty - * - * Marks a page changed, to be written back. - * The caller had better have the write lock on the page via cache_lock() - * before changing its value. - * - * Clockcache already begins a dirty interval when the caller acquires the - * write lock; this call is therefore idempotent there. It remains part of the - * cache API to document the mutation and preserve the contract for other - * implementations. - *---------------------------------------------------------------------- - */ -static inline void -cache_mark_dirty(cache *cc, page_handle *page) -{ - return cc->ops->page_mark_dirty(cc, page); -} - /* *---------------------------------------------------------------------- * cache_pin @@ -560,11 +540,7 @@ cache_flush(cache *cc) * * Wait until every page whose current dirty interval began before this call's * cut has completed writeback. Pages dirtied after the cut do not delay the - * call. The cache implementation must inspect resident metadata only; it must - * not fault pages in merely to satisfy the fence. - * - * This is the checkpointing primitive. It is intentionally narrower than - * cache_flush(), which attempts to leave the entire cache clean. + * call. *----------------------------------------------------------------------------- */ static inline platform_status diff --git a/src/clockcache.c b/src/clockcache.c index 760a2a70..44474e23 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -2477,27 +2477,6 @@ clockcache_unlock(clockcache *cc, page_handle *page) } -/*---------------------------------------------------------------------- - * clockcache_mark_dirty -- - * - * Marks the entry dirty. - *---------------------------------------------------------------------- - */ -void -clockcache_mark_dirty(clockcache *cc, page_handle *page) -{ - debug_only clockcache_entry *entry = clockcache_page_to_entry(cc, page); - uint32 entry_number = clockcache_page_to_entry_number(cc, page); - - clockcache_log(entry->page.disk_addr, - entry_number, - "mark_dirty: entry %u addr %lu\n", - entry_number, - entry->page.disk_addr); - clockcache_dirty_begin(cc, entry_number); - return; -} - /* *---------------------------------------------------------------------- * clockcache_pin -- @@ -3409,13 +3388,6 @@ clockcache_prefetch_page_virtual(cache *c, uint64 addr, page_type type) clockcache_prefetch_page(cc, addr, type); } -void -clockcache_mark_dirty_virtual(cache *c, page_handle *page) -{ - clockcache *cc = (clockcache *)c; - clockcache_mark_dirty(cc, page); -} - void clockcache_pin_virtual(cache *c, page_handle *page) { @@ -3629,7 +3601,6 @@ static cache_ops clockcache_ops = { .page_unlock = clockcache_unlock_virtual, .page_prefetch = clockcache_prefetch_virtual, .page_prefetch_page = clockcache_prefetch_page_virtual, - .page_mark_dirty = clockcache_mark_dirty_virtual, .page_pin = clockcache_pin_virtual, .page_unpin = clockcache_unpin_virtual, .page_writeback = clockcache_page_writeback_virtual, diff --git a/src/core.c b/src/core.c index c8658799..b91d6f2e 100644 --- a/src/core.c +++ b/src/core.c @@ -274,7 +274,6 @@ core_write_checkpoint_page(core_handle *spl, platform_assert(contents_size <= cache_page_size(spl->cc)); memset(page->data, 0, cache_page_size(spl->cc)); memcpy(page->data, contents, contents_size); - cache_mark_dirty(spl->cc, page); cache_unlock(spl->cc, page); cache_unclaim(spl->cc, page); cache_page_writeback(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); @@ -287,7 +286,6 @@ core_initialize_checkpoint_record_page(core_handle *spl, uint64 page_addr) page_handle *page = cache_alloc(spl->cc, page_addr, PAGE_TYPE_SUPERBLOCK); platform_assert(page != NULL); memset(page->data, 0, cache_page_size(spl->cc)); - cache_mark_dirty(spl->cc, page); cache_unlock(spl->cc, page); cache_unclaim(spl->cc, page); cache_page_writeback(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); diff --git a/src/mini_allocator.c b/src/mini_allocator.c index fa350f78..3142a6a7 100644 --- a/src/mini_allocator.c +++ b/src/mini_allocator.c @@ -198,7 +198,6 @@ mini_full_lock_meta_tail(mini_allocator *mini) static void mini_full_unlock_meta_page(mini_allocator *mini, page_handle *meta_page) { - cache_mark_dirty(mini->cc, meta_page); cache_unlock(mini->cc, meta_page); cache_unclaim(mini->cc, meta_page); cache_unget(mini->cc, meta_page); diff --git a/src/shard_log.c b/src/shard_log.c index cccf8afe..205ed3ca 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -434,7 +434,6 @@ shard_log_seal(log_handle *logh) log_entry_set_terminal(cursor); } hdr->checksum = shard_log_checksum(log->cfg, page); - cache_mark_dirty(cc, page); cache_unlock(cc, page); cache_unclaim(cc, page); diff --git a/src/trunk.c b/src/trunk.c index d24c17ae..c95c0c72 100644 --- a/src/trunk.c +++ b/src/trunk.c @@ -2008,7 +2008,6 @@ trunk_node_serialize_maybe_setup_next_page(cache *cc, "%s():%d: cache_alloc() failed", __func__, __LINE__); return STATUS_NO_MEMORY; } - cache_mark_dirty(cc, *current_page); *page_offset = 0; } @@ -2131,7 +2130,6 @@ trunk_node_serialize(trunk_context *context, trunk_node *node) rc = STATUS_NO_MEMORY; goto cleanup; } - cache_mark_dirty(context->cc, header_page); int64 min_inflight_bundle_start = trunk_node_first_live_inflight_bundle(node); diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index d75267d3..5eb6dd72 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -489,7 +489,6 @@ test_cache_writeback_fence(cache *cc, memset(old_page->data, old_value, cache_config_page_size(&cfg->super)); - cache_mark_dirty(cc, old_page); clockcache *clock = (clockcache *)cc; uint64 initial_generation = @@ -528,7 +527,6 @@ test_cache_writeback_fence(cache *cc, memset(new_page->data, new_value, cache_config_page_size(&cfg->super)); - cache_mark_dirty(cc, new_page); cache_unlock(cc, new_page); new_locked = FALSE; cache_unclaim(cc, new_page); @@ -924,7 +922,6 @@ test_cache_basic(cache *cc, clockcache_config *cfg, platform_heap_id hid) } } for (i = 0; i < cfg->page_capacity; i++) { - cache_mark_dirty(cc, page_arr[i]); cache_unlock(cc, page_arr[i]); cache_unclaim(cc, page_arr[i]); cache_unget(cc, page_arr[i]); @@ -1110,7 +1107,6 @@ cache_test_dirty_flush(cache *cc, rc = STATUS_TEST_FAILED; break; } - cache_mark_dirty(cc, ph); cache_unlock(cc, ph); cache_unclaim(cc, ph); cache_unget(cc, ph); From 7aebf2c8ad61f5e108a1dcaa9d3ffaf9d501619c Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 17 Jul 2026 16:44:35 -0700 Subject: [PATCH 04/64] fix cach writeback impl Signed-off-by: Rob Johnson --- src/clockcache.c | 317 ++++++---------------------------- src/clockcache.h | 15 +- tests/functional/cache_test.c | 169 ++++++++---------- 3 files changed, 130 insertions(+), 371 deletions(-) diff --git a/src/clockcache.c b/src/clockcache.c index 44474e23..10ceb2e7 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -141,9 +141,6 @@ clockcache_print(platform_log_handle *log_handle, clockcache *cc); #define CC_LOADING (1u << 4) // page is actively being read from disk #define CC_WRITELOCKED (1u << 5) // write lock is held #define CC_CLAIMED (1u << 6) // claim is held -// A checkpoint fence has selected this dirty interval for writeback. New -// writers must not claim the page until that writeback completes. -#define CC_WRITEBACK_REQUESTED (1u << 7) /* Common status flag combinations */ // free entry @@ -227,66 +224,25 @@ clockcache_test_flag(clockcache *cc, uint32 entry_number, entry_status flag) /* *-------------------------------------------------------------------------- - * Dirty-generation bookkeeping - * - * A dirty generation identifies the clean->dirty interval, rather than every - * individual mutation. A checkpoint fence rotates the current generation and - * drains all older intervals. Writers cannot modify a page during writeback, - * so writing a later version of an older dirty interval still satisfies the - * fence. + * clockcache_dirty_complete_writeback -- + * + * Mark an entry clean once its writeback has completed. CC_CLEAN must be set + * before CC_WRITEBACK is cleared: the intermediate CC_CLEAN|CC_WRITEBACK state + * is not cleanable, so no thread can start a duplicate writeback in the gap. A + * write lock cannot be held while CC_WRITEBACK is set (see + * clockcache_get_write), so the clean<->dirty transitions never race and no + * lock is needed here. *-------------------------------------------------------------------------- */ -static void -clockcache_dirty_begin(clockcache *cc, uint32 entry_number) -{ - platform_status rc = platform_mutex_lock(&cc->dirty_lock); - platform_assert_status_ok(rc); - - if (clockcache_test_flag(cc, entry_number, CC_CLEAN)) { - clockcache_entry *entry = clockcache_get_entry(cc, entry_number); - debug_assert(entry->dirty_generation == 0); - entry->dirty_generation = - __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); - clockcache_clear_flag(cc, entry_number, CC_CLEAN); - } - - rc = platform_mutex_unlock(&cc->dirty_lock); - platform_assert_status_ok(rc); -} - static void clockcache_dirty_complete_writeback(clockcache *cc, uint32 entry_number) { - platform_status rc = platform_mutex_lock(&cc->dirty_lock); - platform_assert_status_ok(rc); - - clockcache_entry *entry = clockcache_get_entry(cc, entry_number); - debug_assert(entry->dirty_generation != 0); - entry->dirty_generation = 0; debug_only uint32 was_clean = clockcache_set_flag(cc, entry_number, CC_CLEAN); debug_assert(!was_clean); debug_only uint32 was_writeback = clockcache_clear_flag(cc, entry_number, CC_WRITEBACK); debug_assert(was_writeback); - clockcache_clear_flag(cc, entry_number, CC_WRITEBACK_REQUESTED); - - rc = platform_mutex_unlock(&cc->dirty_lock); - platform_assert_status_ok(rc); -} - -static void -clockcache_dirty_discard(clockcache *cc, uint32 entry_number) -{ - platform_status rc = platform_mutex_lock(&cc->dirty_lock); - platform_assert_status_ok(rc); - - clockcache_entry *entry = clockcache_get_entry(cc, entry_number); - entry->dirty_generation = 0; - clockcache_clear_flag(cc, entry_number, CC_WRITEBACK_REQUESTED); - - rc = platform_mutex_unlock(&cc->dirty_lock); - platform_assert_status_ok(rc); } #ifdef RECORD_ACQUISITION_STACKS @@ -694,27 +650,11 @@ clockcache_try_get_claim(clockcache *cc, uint32 entry_number) entry_number, clockcache_test_flag(cc, entry_number, CC_CLAIMED)); - if (clockcache_test_flag(cc, entry_number, CC_WRITEBACK_REQUESTED)) { - return GET_RC_CONFLICT; - } - if (clockcache_set_flag(cc, entry_number, CC_CLAIMED)) { clockcache_log(0, entry_number, "return false\n", NULL); return GET_RC_CONFLICT; } - /* - * A fence may have selected the dirty interval between the first test and - * the claim. Do not let a new writer keep that selected interval dirty; - * drop the claim and let the requested writeback proceed instead. - */ - if (clockcache_test_flag(cc, entry_number, CC_WRITEBACK_REQUESTED)) { - debug_only uint32 was_claimed = - clockcache_clear_flag(cc, entry_number, CC_CLAIMED); - debug_assert(was_claimed); - return GET_RC_CONFLICT; - } - clockcache_record_backtrace(cc, entry_number); return GET_RC_SUCCESS; @@ -865,16 +805,11 @@ clockcache_try_get_write(clockcache *cc, uint32 entry_number) static inline bool32 clockcache_ok_to_writeback(clockcache *cc, uint32 entry_number, - bool32 with_access, - bool32 requested_only) + bool32 with_access) { uint32 status = clockcache_get_status(cc, entry_number); - uint32 request_flag = status & CC_WRITEBACK_REQUESTED; - uint32 base_status = status & ~CC_WRITEBACK_REQUESTED; - - return ((!requested_only || request_flag) - && ((base_status == CC_CLEANABLE1_STATUS) - || (with_access && base_status == CC_CLEANABLE2_STATUS))); + return (status == CC_CLEANABLE1_STATUS) + || (with_access && status == CC_CLEANABLE2_STATUS); } /* @@ -900,24 +835,14 @@ clockcache_try_set_writeback(clockcache *cc, volatile uint32 *status = &cc->entry[entry_number].status; if (__sync_bool_compare_and_swap( - status, CC_CLEANABLE1_STATUS, CC_WRITEBACK1_STATUS) - || __sync_bool_compare_and_swap(status, - CC_CLEANABLE1_STATUS - | CC_WRITEBACK_REQUESTED, - CC_WRITEBACK1_STATUS - | CC_WRITEBACK_REQUESTED)) + status, CC_CLEANABLE1_STATUS, CC_WRITEBACK1_STATUS)) { return TRUE; } if (with_access - && (__sync_bool_compare_and_swap( - status, CC_CLEANABLE2_STATUS, CC_WRITEBACK2_STATUS) - || __sync_bool_compare_and_swap(status, - CC_CLEANABLE2_STATUS - | CC_WRITEBACK_REQUESTED, - CC_WRITEBACK2_STATUS - | CC_WRITEBACK_REQUESTED))) + && __sync_bool_compare_and_swap( + status, CC_CLEANABLE2_STATUS, CC_WRITEBACK2_STATUS)) { return TRUE; } @@ -1016,10 +941,7 @@ clockcache_abort_writeback_range(clockcache *cc, *---------------------------------------------------------------------- */ static platform_status -clockcache_batch_start_writeback(clockcache *cc, - uint64 batch, - bool32 is_urgent, - bool32 requested_only) +clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) { uint32 entry_no, next_entry_no; uint64 addr, first_addr, end_addr, i; @@ -1050,8 +972,7 @@ clockcache_batch_start_writeback(clockcache *cc, entry = &cc->entry[entry_no]; addr = entry->page.disk_addr; // test and test and set in the if condition - if (clockcache_ok_to_writeback( - cc, entry_no, is_urgent, requested_only) + if (clockcache_ok_to_writeback(cc, entry_no, is_urgent) && clockcache_try_set_writeback(cc, entry_no, is_urgent)) { debug_assert(clockcache_lookup(cc, addr) == entry_no); @@ -1066,8 +987,7 @@ clockcache_batch_start_writeback(clockcache *cc, next_entry_no = CC_UNMAPPED_ENTRY; } while ( next_entry_no != CC_UNMAPPED_ENTRY - && clockcache_ok_to_writeback( - cc, next_entry_no, is_urgent, requested_only) + && clockcache_ok_to_writeback(cc, next_entry_no, is_urgent) && clockcache_try_set_writeback(cc, next_entry_no, is_urgent)); first_addr += page_size; end_addr = entry->page.disk_addr; @@ -1081,8 +1001,7 @@ clockcache_batch_start_writeback(clockcache *cc, next_entry_no = CC_UNMAPPED_ENTRY; } while ( next_entry_no != CC_UNMAPPED_ENTRY - && clockcache_ok_to_writeback( - cc, next_entry_no, is_urgent, requested_only) + && clockcache_ok_to_writeback(cc, next_entry_no, is_urgent) && clockcache_try_set_writeback(cc, next_entry_no, is_urgent)); @@ -1170,100 +1089,21 @@ clockcache_batch_start_writeback(clockcache *cc, return result; } -/* - *-------------------------------------------------------------------------- - * clockcache_fence_request_old_dirty -- - * - * Mark every dirty interval at or before cutoff for writeback. The dirty lock - * makes this atomic with both a clean->dirty transition and a writeback - * completion, so a false return means no interval from this fence remains. - *-------------------------------------------------------------------------- - */ -static bool32 -clockcache_fence_request_old_dirty(clockcache *cc, uint64 cutoff) -{ - bool32 pending = FALSE; - platform_status rc = platform_mutex_lock(&cc->dirty_lock); - platform_assert_status_ok(rc); - - for (uint32 entry_no = 0; entry_no < cc->cfg->page_capacity; entry_no++) { - clockcache_entry *entry = clockcache_get_entry(cc, entry_no); - if (entry->dirty_generation != 0 - && entry->dirty_generation <= cutoff) - { - clockcache_set_flag(cc, entry_no, CC_WRITEBACK_REQUESTED); - pending = TRUE; - } - } - - rc = platform_mutex_unlock(&cc->dirty_lock); - platform_assert_status_ok(rc); - return pending; -} - -/* - *-------------------------------------------------------------------------- - * clockcache_fence_cancel_requests -- - * - * Abandon a failed fence without leaving writers permanently blocked behind - * its request bits. A later fence will select these still-dirty generations - * again. - *-------------------------------------------------------------------------- - */ -static void -clockcache_fence_cancel_requests(clockcache *cc, uint64 cutoff) -{ - platform_status rc = platform_mutex_lock(&cc->dirty_lock); - platform_assert_status_ok(rc); - - for (uint32 entry_no = 0; entry_no < cc->cfg->page_capacity; entry_no++) { - clockcache_entry *entry = clockcache_get_entry(cc, entry_no); - if (entry->dirty_generation != 0 - && entry->dirty_generation <= cutoff) - { - clockcache_clear_flag(cc, entry_no, CC_WRITEBACK_REQUESTED); - } - } - - rc = platform_mutex_unlock(&cc->dirty_lock); - platform_assert_status_ok(rc); -} - -/* - *-------------------------------------------------------------------------- - * clockcache_fence_start_writeback_round -- - * - * Start writeback only for intervals selected by a checkpoint fence. This is - * a metadata scan of resident cache entries; no page lookup or trunk walk is - * involved. The normal clock hand may retain batch_busy while it owns a free - * batch, so a fence intentionally does not wait for that batch-level state. - * Per-entry compare-and-swap on CC_WRITEBACK provides the required exclusion - * from the normal cleaner. - *-------------------------------------------------------------------------- - */ -static platform_status -clockcache_fence_start_writeback_round(clockcache *cc) -{ - for (uint64 batch = 0; batch < cc->cfg->batch_capacity; batch++) { - platform_status rc = - clockcache_batch_start_writeback(cc, batch, TRUE, TRUE); - if (!SUCCESS(rc)) { - return rc; - } - } - return STATUS_OK; -} - /* *-------------------------------------------------------------------------- * clockcache_writeback_fence -- * - * Rotate the dirty generation, then wait only for dirty intervals that were - * already present at that cut. Pages dirtied after the cut carry the next - * generation and do not delay this checkpoint. A request bit blocks new - * writers from indefinitely extending a selected interval, while a writer - * that was already active is allowed to finish and its final bytes are what - * gets written. + * Issue writeback for every dirty, unlocked page and wait for all of it to + * complete, so that a subsequent durable_barrier can make it durable. + * + * A single pass over the cache suffices. Everything reachable from a published + * checkpoint root is copy-on-write and therefore never write-locked while it is + * reachable, so a dirty page that a writer currently holds locked cannot be + * part of this checkpoint and is safely skipped (clockcache_ok_to_writeback + * rejects locked pages). Pages that a concurrent cleaner already has in + * writeback are not re-issued here but are still drained by the final + * io_wait_all. The fence lock serializes checkpoints so that only one thread + * drains the shared I/O contexts at a time. *-------------------------------------------------------------------------- */ platform_status @@ -1272,31 +1112,21 @@ clockcache_writeback_fence(clockcache *cc) platform_status rc = platform_mutex_lock(&cc->writeback_fence_lock); platform_assert_status_ok(rc); - rc = platform_mutex_lock(&cc->dirty_lock); - platform_assert_status_ok(rc); - uint64 cutoff = __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); - platform_assert(cutoff < UINT64_MAX); - __atomic_store_n(&cc->dirty_generation, cutoff + 1, __ATOMIC_RELEASE); - rc = platform_mutex_unlock(&cc->dirty_lock); - platform_assert_status_ok(rc); - - while (clockcache_fence_request_old_dirty(cc, cutoff)) { - rc = clockcache_fence_start_writeback_round(cc); - if (!SUCCESS(rc)) { - clockcache_fence_cancel_requests(cc, cutoff); + platform_status result = STATUS_OK; + for (uint64 batch = 0; batch < cc->cfg->batch_capacity; batch++) { + result = clockcache_batch_start_writeback(cc, batch, TRUE); + if (!SUCCESS(result)) { break; } - /* - * Poll the local completion queue, then yield briefly if all selected - * pages are currently write-locked or in another cleaner's I/O. - */ - clockcache_wait(cc); - platform_sleep_ns(1000); } + // Drain everything issued above (and any in-flight cleaner writeback) so the + // caller's durable_barrier sees completed writes. + io_wait_all(cc->io); + platform_status unlock_rc = platform_mutex_unlock(&cc->writeback_fence_lock); platform_assert_status_ok(unlock_rc); - return rc; + return result; } /* @@ -1397,7 +1227,6 @@ clockcache_try_evict(clockcache *cc, uint32 entry_number) /* 6. set status to CC_FREE_STATUS (clears claim and write lock) */ platform_assert(entry->waiters.head == NULL); entry->type = PAGE_TYPE_INVALID; - clockcache_dirty_discard(cc, entry_number); entry->status = CC_FREE_STATUS; clockcache_log( addr, entry_number, "evict: entry %u addr %lu\n", entry_number, addr); @@ -1478,8 +1307,8 @@ clockcache_move_hand(clockcache *cc, bool32 is_urgent) cleaner_hand = (evict_hand + cc->cleaner_gap) % cc->cfg->batch_capacity; clean_batch_busy = &cc->batch_busy[cleaner_hand]; if (__sync_bool_compare_and_swap(clean_batch_busy, FALSE, TRUE)) { - platform_status rc = clockcache_batch_start_writeback( - cc, cleaner_hand, is_urgent, FALSE); + platform_status rc = + clockcache_batch_start_writeback(cc, cleaner_hand, is_urgent); if (!SUCCESS(rc)) { platform_error_log("clockcache_move_hand: writeback failed: %s\n", platform_status_to_string(rc)); @@ -1506,8 +1335,7 @@ clockcache_get_free_page(clockcache *cc, uint32 status, page_type type, bool32 refcount, - bool32 blocking, - bool32 begins_dirty) + bool32 blocking) { uint32 entry_no; uint64 num_passes = 0; @@ -1517,7 +1345,6 @@ clockcache_get_free_page(clockcache *cc, timestamp wait_start; debug_assert((tid < MAX_THREADS), "Invalid tid=%lu\n", tid); - debug_assert(!begins_dirty || status == CC_ALLOC_STATUS); if (cc->per_thread[tid].free_hand == CC_UNMAPPED_ENTRY) { clockcache_move_hand(cc, FALSE); } @@ -1533,40 +1360,15 @@ clockcache_get_free_page(clockcache *cc, for (entry_no = start_entry; entry_no < end_entry; entry_no++) { entry = &cc->entry[entry_no]; if (entry->status == CC_FREE_STATUS) { - platform_status rc = STATUS_OK; - if (begins_dirty) { - rc = platform_mutex_lock(&cc->dirty_lock); - platform_assert_status_ok(rc); - } - bool32 reserved = __sync_bool_compare_and_swap( &entry->status, CC_FREE_STATUS, CC_ALLOC_STATUS); if (!reserved) { - if (begins_dirty) { - rc = platform_mutex_unlock(&cc->dirty_lock); - platform_assert_status_ok(rc); - } continue; } - /* - * The allocation's dirty interval is tagged while holding the - * same lock as a fence cut. This prevents a checkpoint from - * seeing a dirty CC_ALLOC_STATUS entry with generation zero. - */ - debug_assert(entry->dirty_generation == 0); - if (begins_dirty) { - entry->dirty_generation = - __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); - } entry->status = status; entry->type = type; - if (begins_dirty) { - rc = platform_mutex_unlock(&cc->dirty_lock); - platform_assert_status_ok(rc); - } - if (refcount) { clockcache_inc_ref(cc, entry_no, tid); } @@ -1630,7 +1432,7 @@ clockcache_flush(clockcache *cc) flush_hand++) { platform_status rc = - clockcache_batch_start_writeback(cc, flush_hand, TRUE, FALSE); + clockcache_batch_start_writeback(cc, flush_hand, TRUE); platform_assert_status_ok(rc); } @@ -1692,8 +1494,7 @@ clockcache_alloc(clockcache *cc, uint64 addr, page_type type) CC_ALLOC_STATUS, type, TRUE, // refcount - TRUE, // blocking - TRUE); // begins_dirty + TRUE); // blocking clockcache_entry *entry = &cc->entry[entry_no]; entry->page.disk_addr = addr; entry->type = type; @@ -1793,7 +1594,6 @@ clockcache_try_page_discard(clockcache *cc, uint64 addr) /* 6. set status to CC_FREE_STATUS (clears claim and write lock) */ platform_assert(entry->waiters.head == NULL); entry->type = PAGE_TYPE_INVALID; - clockcache_dirty_discard(cc, entry_number); entry->status = CC_FREE_STATUS; /* 7. reset pincount */ @@ -1928,8 +1728,7 @@ clockcache_acquire_entry_for_load(clockcache *cc, // IN CC_READ_LOADING_STATUS, type, TRUE, // refcount - TRUE, // blocking - FALSE); // begins_dirty + TRUE); // blocking clockcache_entry *entry = clockcache_get_entry(cc, entry_number); /* * If someone else is loading the page and has reserved the lookup, let them @@ -2457,7 +2256,9 @@ clockcache_lock(clockcache *cc, page_handle *page) entry_number, page->disk_addr); clockcache_get_write(cc, entry_number); - clockcache_dirty_begin(cc, entry_number); + // A write lock marks the page dirty; the CC_WRITEBACK exclusion in + // clockcache_get_write guarantees this cannot race a writeback completion. + clockcache_clear_flag(cc, entry_number, CC_CLEAN); } void @@ -2886,7 +2687,7 @@ clockcache_prefetch_pages(clockcache *cc, case GET_RC_EVICTED: { uint32 free_entry_no = clockcache_get_free_page( - cc, CC_READ_LOADING_STATUS, type, FALSE, TRUE, FALSE); + cc, CC_READ_LOADING_STATUS, type, FALSE, TRUE); clockcache_entry *entry = &cc->entry[free_entry_no]; entry->page.disk_addr = addr; entry->type = type; @@ -3700,24 +3501,15 @@ clockcache_init(clockcache *cc, // OUT cc->io = io; cc->heap_id = hid; - platform_status rc = platform_mutex_init(&cc->dirty_lock, mid, hid); - if (!SUCCESS(rc)) { - platform_error_log("clockcache_init: failed to initialize dirty lock: %s\n", - platform_status_to_string(rc)); - goto alloc_error; - } - - rc = platform_mutex_init(&cc->writeback_fence_lock, mid, hid); + platform_status rc = + platform_mutex_init(&cc->writeback_fence_lock, mid, hid); if (!SUCCESS(rc)) { platform_error_log( "clockcache_init: failed to initialize writeback fence lock: %s\n", platform_status_to_string(rc)); - platform_status destroy_rc = platform_mutex_destroy(&cc->dirty_lock); - platform_assert_status_ok(destroy_rc); goto alloc_error; } - cc->dirty_generation = 1; - cc->dirty_locks_initialized = TRUE; + cc->writeback_fence_lock_initialized = TRUE; /* lookup maps addrs to entries, entry contains the entries themselves */ rc = platform_buffer_init(&cc->lookup_bh, @@ -3762,7 +3554,6 @@ clockcache_init(clockcache *cc, // OUT cc->data + clockcache_multiply_by_page_size(cc, i); cc->entry[i].page.disk_addr = CC_UNMAPPED_ADDR; cc->entry[i].status = CC_FREE_STATUS; - cc->entry[i].dirty_generation = 0; cc->entry[i].type = PAGE_TYPE_INVALID; async_wait_queue_init(&cc->entry[i].waiters); } @@ -3845,12 +3636,10 @@ clockcache_deinit(clockcache *cc) // IN/OUT #endif } - if (cc->dirty_locks_initialized) { + if (cc->writeback_fence_lock_initialized) { rc = platform_mutex_destroy(&cc->writeback_fence_lock); platform_assert_status_ok(rc); - rc = platform_mutex_destroy(&cc->dirty_lock); - platform_assert_status_ok(rc); - cc->dirty_locks_initialized = FALSE; + cc->writeback_fence_lock_initialized = FALSE; } if (cc->lookup) { diff --git a/src/clockcache.h b/src/clockcache.h index 1e601bd5..0ef394a4 100644 --- a/src/clockcache.h +++ b/src/clockcache.h @@ -73,9 +73,6 @@ typedef uint32 entry_status; // Saved in clockcache_entry->status struct clockcache_entry { page_handle page; volatile entry_status status; - // Generation in which this page's current dirty interval began. Zero means - // no dirty interval (a clean page or an unpublished clean-load reservation). - volatile uint64 dirty_generation; page_type type; async_wait_queue waiters; #ifdef RECORD_ACQUISITION_STACKS @@ -143,16 +140,10 @@ struct clockcache { volatile bool32 *batch_busy; // Convenience pointer for batch_bh uint64 cleaner_gap; - /* - * Dirty-generation bookkeeping for checkpoint writeback fences. The dirty - * lock serializes the clean<->dirty transition with a fence cut and with - * writeback completion. Fence rounds are serialized separately so a page - * only needs one outstanding writeback-request bit. - */ - platform_mutex dirty_lock; + // Serializes checkpoint writeback fences so only one thread drains the + // shared I/O contexts at a time (see clockcache_writeback_fence). platform_mutex writeback_fence_lock; - uint64 dirty_generation; - bool32 dirty_locks_initialized; + bool32 writeback_fence_lock_initialized; volatile struct { volatile uint32 free_hand; diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index 5eb6dd72..f6ce1d75 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -411,9 +411,6 @@ typedef struct { platform_status status; } cache_fence_test_context; -/* Mirrors clockcache's private CC_WRITEBACK_REQUESTED status bit. */ -#define CACHE_TEST_WRITEBACK_REQUESTED (1u << 7) - static void cache_test_writeback_fence_thread(void *arg) { @@ -423,10 +420,11 @@ cache_test_writeback_fence_thread(void *arg) } /* - * Verify that a fence drains an already-dirty page without waiting for a page - * dirtied after its cut. Keeping old_page write-locked forces the fence to - * remain in progress long enough to deterministically dirty new_page after - * the dirty-generation rotation. + * Verify the simplified writeback-fence contract: a single pass flushes every + * dirty, unlocked page and completes even while another page is write-locked, + * skipping the locked page rather than waiting for it. old_page is left dirty + * and unlocked so the fence must flush it; new_page is held write-locked so the + * fence must skip it and still finish promptly. */ static platform_status test_cache_writeback_fence(cache *cc, @@ -439,7 +437,6 @@ test_cache_writeback_fence(cache *cc, uint64 *addr_arr = NULL; page_handle *old_page = NULL; page_handle *new_page = NULL; - bool32 old_claimed = FALSE, old_locked = FALSE; bool32 new_claimed = FALSE, new_locked = FALSE; bool32 fence_thread_started = FALSE; const uint8 old_baseline = 0x11, old_value = 0x22; @@ -478,21 +475,30 @@ test_cache_writeback_fence(cache *cc, } cache_flush(cc); + /* Dirty old_page and leave it unlocked: the fence must flush it. */ old_page = cache_get(cc, addr_arr[0], TRUE, PAGE_TYPE_MISC); if (!cache_try_claim(cc, old_page)) { rc = STATUS_TEST_FAILED; goto cleanup; } - old_claimed = TRUE; cache_lock(cc, old_page); - old_locked = TRUE; - memset(old_page->data, - old_value, - cache_config_page_size(&cfg->super)); - - clockcache *clock = (clockcache *)cc; - uint64 initial_generation = - __atomic_load_n(&clock->dirty_generation, __ATOMIC_ACQUIRE); + memset(old_page->data, old_value, cache_config_page_size(&cfg->super)); + cache_unlock(cc, old_page); + cache_unclaim(cc, old_page); + cache_unget(cc, old_page); + old_page = NULL; + + /* Hold new_page write-locked: the fence must skip it and still complete. */ + new_page = cache_get(cc, addr_arr[pages_per_extent], TRUE, PAGE_TYPE_MISC); + if (!cache_try_claim(cc, new_page)) { + rc = STATUS_TEST_FAILED; + goto cleanup; + } + new_claimed = TRUE; + cache_lock(cc, new_page); + new_locked = TRUE; + memset(new_page->data, new_value, cache_config_page_size(&cfg->super)); + rc = platform_thread_create(&fence_thread, FALSE, cache_test_writeback_fence_thread, @@ -503,43 +509,18 @@ test_cache_writeback_fence(cache *cc, } fence_thread_started = TRUE; + /* The fence must finish without waiting for the write-locked new_page. */ timestamp wait_start = platform_get_timestamp(); - while (__atomic_load_n(&clock->dirty_generation, __ATOMIC_ACQUIRE) - == initial_generation) - { + while (!__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { if (platform_timestamp_elapsed(wait_start) > SEC_TO_NSEC(5)) { - platform_error_log("cache_test: writeback fence did not take a cut\n"); + platform_error_log( + "cache_test: fence did not complete while a page was locked\n"); rc = STATUS_TIMEDOUT; goto cleanup; } platform_sleep_ns(1000); } - new_page = cache_get( - cc, addr_arr[pages_per_extent], TRUE, PAGE_TYPE_MISC); - if (!cache_try_claim(cc, new_page)) { - rc = STATUS_TEST_FAILED; - goto cleanup; - } - new_claimed = TRUE; - cache_lock(cc, new_page); - new_locked = TRUE; - memset(new_page->data, - new_value, - cache_config_page_size(&cfg->super)); - cache_unlock(cc, new_page); - new_locked = FALSE; - cache_unclaim(cc, new_page); - new_claimed = FALSE; - cache_unget(cc, new_page); - new_page = NULL; - - if (__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { - platform_error_log("cache_test: fence completed while old page was locked\n"); - rc = STATUS_TEST_FAILED; - goto cleanup; - } - cleanup: if (new_locked) { cache_unlock(cc, new_page); @@ -550,12 +531,6 @@ test_cache_writeback_fence(cache *cc, if (new_page != NULL) { cache_unget(cc, new_page); } - if (old_locked) { - cache_unlock(cc, old_page); - } - if (old_claimed) { - cache_unclaim(cc, old_page); - } if (old_page != NULL) { cache_unget(cc, old_page); } @@ -569,10 +544,11 @@ test_cache_writeback_fence(cache *cc, } } + /* Only new_page, write-locked throughout the fence, remains dirty. */ if (SUCCESS(rc)) { uint32 dirty_count = cache_count_dirty(cc); if (dirty_count != 1) { - platform_error_log("cache_test: expected one post-cut dirty page, " + platform_error_log("cache_test: expected one dirty page after fence, " "found %u\n", dirty_count); rc = STATUS_TEST_FAILED; @@ -581,14 +557,15 @@ test_cache_writeback_fence(cache *cc, if (SUCCESS(rc)) { rc = cache_durable_barrier(cc); } + /* The dirty, unlocked page was flushed by the fence. */ if (SUCCESS(rc)) { rc = cache_test_verify_disk_page(cc, addr_arr[0], cache_config_page_size(&cfg->super), old_value); } + /* The write-locked page was skipped, so its baseline is still on disk. */ if (SUCCESS(rc)) { - /* The post-cut page remains dirty, so its baseline must still be disk. */ rc = cache_test_verify_disk_page(cc, addr_arr[pages_per_extent], cache_config_page_size(&cfg->super), @@ -618,8 +595,9 @@ test_cache_writeback_fence(cache *cc, } /* - * A fence must allow an already-claimed writer to finish its selected dirty - * interval, but it must not complete before that writer releases the claim. + * A page that is only claimed (not write-locked) is not cleanable, so a fence + * skips it and completes; the page's baseline is therefore still what is on + * disk. Once the claim is dropped, a second fence flushes the page. */ static platform_status test_cache_writeback_fence_claimed_page(cache *cc, @@ -657,6 +635,7 @@ test_cache_writeback_fence_claimed_page(cache *cc, } cache_flush(cc); + /* Dirty the page, then keep only a claim (release the write lock). */ page = cache_get(cc, addr_arr[0], TRUE, PAGE_TYPE_MISC); if (!cache_try_claim(cc, page)) { rc = STATUS_TEST_FAILED; @@ -669,9 +648,6 @@ test_cache_writeback_fence_claimed_page(cache *cc, cache_unlock(cc, page); locked = FALSE; - clockcache *clock = (clockcache *)cc; - uint64 initial_generation = - __atomic_load_n(&clock->dirty_generation, __ATOMIC_ACQUIRE); rc = platform_thread_create(&fence_thread, FALSE, cache_test_writeback_fence_thread, @@ -682,41 +658,58 @@ test_cache_writeback_fence_claimed_page(cache *cc, } fence_thread_started = TRUE; - uint64 entry_no = - clock->lookup[addr_arr[0] >> clock->cfg->log_page_size]; - platform_assert(entry_no < clock->cfg->page_capacity); + /* A claimed page is not cleanable, so the fence skips it and completes. */ timestamp wait_start = platform_get_timestamp(); - while ((__atomic_load_n(&clock->dirty_generation, __ATOMIC_ACQUIRE) - == initial_generation) - || !(__atomic_load_n(&clock->entry[entry_no].status, - __ATOMIC_ACQUIRE) - & CACHE_TEST_WRITEBACK_REQUESTED)) - { + while (!__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { if (platform_timestamp_elapsed(wait_start) > SEC_TO_NSEC(5)) { - platform_error_log("cache_test: fence did not request claimed page\n"); + platform_error_log( + "cache_test: fence did not complete while page was claimed\n"); rc = STATUS_TIMEDOUT; goto cleanup; } platform_sleep_ns(1000); } - if (__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { - platform_error_log("cache_test: fence completed while page was claimed\n"); - rc = STATUS_TEST_FAILED; + platform_status join_rc = platform_thread_join(&fence_thread); + fence_thread_started = FALSE; + if (!SUCCESS(join_rc)) { + rc = join_rc; + goto cleanup; + } + if (!SUCCESS(fence_ctxt.status)) { + rc = fence_ctxt.status; goto cleanup; } - /* This claim predates the request, so it is permitted to finish. */ - cache_lock(cc, page); - locked = TRUE; - memset(page->data, final_value, cache_config_page_size(&cfg->super)); - cache_unlock(cc, page); - locked = FALSE; + /* The skipped page was not written, so its baseline is still on disk. */ + rc = cache_test_verify_disk_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + + /* Drop the claim; the page is now cleanable and a second fence flushes it. */ cache_unclaim(cc, page); claimed = FALSE; cache_unget(cc, page); page = NULL; + rc = cache_writeback_fence(cc); + if (!SUCCESS(rc)) { + goto cleanup; + } + if (cache_count_dirty(cc) != 0) { + platform_error_log("cache_test: second fence left dirty pages\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + rc = cache_durable_barrier(cc); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc = cache_test_verify_disk_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), final_value); + cleanup: if (locked) { cache_unlock(cc, page); @@ -728,29 +721,15 @@ test_cache_writeback_fence_claimed_page(cache *cc, cache_unget(cc, page); } if (fence_thread_started) { - platform_status join_rc = platform_thread_join(&fence_thread); - if (!SUCCESS(join_rc) && SUCCESS(rc)) { - rc = join_rc; + platform_status jrc = platform_thread_join(&fence_thread); + if (!SUCCESS(jrc) && SUCCESS(rc)) { + rc = jrc; } if (!SUCCESS(fence_ctxt.status) && SUCCESS(rc)) { rc = fence_ctxt.status; } } - if (SUCCESS(rc) && cache_count_dirty(cc) != 0) { - platform_error_log("cache_test: claimed-page fence left dirty pages\n"); - rc = STATUS_TEST_FAILED; - } - if (SUCCESS(rc)) { - rc = cache_durable_barrier(cc); - } - if (SUCCESS(rc)) { - rc = cache_test_verify_disk_page(cc, - addr_arr[0], - cache_config_page_size(&cfg->super), - final_value); - } - if (addr_arr != NULL) { cache_flush(cc); allocator *al = cache_get_allocator(cc); From c8d0fd8ff5f58482f375f8695970324fbe43094f Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 17 Jul 2026 23:00:50 -0700 Subject: [PATCH 05/64] simplify cache writeback Signed-off-by: Rob Johnson --- src/cache.h | 23 ++- src/clockcache.c | 267 ++++++++++++++++++++------------ src/clockcache.h | 13 +- src/core.c | 178 +++++++++++----------- src/shard_log.c | 2 +- src/trunk.c | 2 +- tests/functional/cache_test.c | 277 ++++++++++++++++++++++++++-------- tests/functional/log_test.c | 102 +++++-------- 8 files changed, 530 insertions(+), 334 deletions(-) diff --git a/src/cache.h b/src/cache.h index 1cecd131..8e37827b 100644 --- a/src/cache.h +++ b/src/cache.h @@ -98,7 +98,7 @@ typedef void (*cache_generic_fn)(cache *cc); typedef uint64 (*cache_generic_uint64_fn)(cache *cc); typedef void (*page_generic_fn)(cache *cc, page_handle *page); typedef platform_status (*cache_durable_barrier_fn)(cache *cc); -typedef platform_status (*cache_writeback_fence_fn)(cache *cc); +typedef platform_status (*cache_writeback_dirty_fn)(cache *cc); typedef page_handle *(*page_alloc_fn)(cache *cc, uint64 addr, page_type type); typedef void (*extent_discard_fn)(cache *cc, uint64 addr, page_type type); @@ -177,7 +177,7 @@ typedef struct cache_ops { page_writeback_fn page_writeback; extent_writeback_fn extent_writeback; cache_generic_fn flush; - cache_writeback_fence_fn writeback_fence; + cache_writeback_dirty_fn writeback_dirty; cache_durable_barrier_fn durable_barrier; evict_fn evict; cache_generic_fn cleanup; @@ -369,11 +369,7 @@ cache_unclaim(cache *cc, page_handle *page) * * Blocks until outstanding read locks are released by other threads. * - * Acquiring the write lock marks the page dirty: it begins a dirty interval on - * this transition (before the caller's first change), so a checkpoint fence - * cannot miss the modification. A page obtained write-locked from cache_alloc() - * is likewise already dirty. Callers therefore do not separately declare the - * mutation. + * Acquiring the write lock marks the page dirty. *---------------------------------------------------------------------- */ static inline void @@ -536,17 +532,16 @@ cache_flush(cache *cc) /* *----------------------------------------------------------------------------- - * cache_writeback_fence + * cache_writeback_dirty * - * Wait until every page whose current dirty interval began before this call's - * cut has completed writeback. Pages dirtied after the cut do not delay the - * call. + * Issues and wait for completion of writebacks for all pages that are dirty + * but not locked at the time of the call. May writeback other pages, as well. *----------------------------------------------------------------------------- */ static inline platform_status -cache_writeback_fence(cache *cc) +cache_writeback_dirty(cache *cc) { - return cc->ops->writeback_fence(cc); + return cc->ops->writeback_dirty(cc); } /* @@ -554,7 +549,7 @@ cache_writeback_fence(cache *cc) * cache_durable_barrier * * Ensure that writeback completed before this call is durable across a power - * loss. Callers normally use this after cache_writeback_fence(), and again + * loss. Callers normally use this after cache_writeback_dirty(), and again * after publishing a checkpoint superblock. *----------------------------------------------------------------------------- */ diff --git a/src/clockcache.c b/src/clockcache.c index 10ceb2e7..cfa48e34 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -224,19 +224,40 @@ clockcache_test_flag(clockcache *cc, uint32 entry_number, entry_status flag) /* *-------------------------------------------------------------------------- - * clockcache_dirty_complete_writeback -- - * + * Dirty-generation bookkeeping + * + * Each entry records the generation in which its current dirty interval began + * (0 when clean/free). A writeback fence takes a "cut" of the generation + * counter and drains every entry stamped at or below that cut. These + * transitions need no dedicated lock: clockcache_dirty_begin runs under the + * page's write lock, and a write lock cannot be held while CC_WRITEBACK is set + * (see clockcache_get_write), so a clean->dirty transition never races a + * writeback completion on the same entry. + *-------------------------------------------------------------------------- + */ +static void +clockcache_dirty_begin(clockcache *cc, uint32 entry_number) +{ + if (clockcache_test_flag(cc, entry_number, CC_CLEAN)) { + clockcache_entry *entry = clockcache_get_entry(cc, entry_number); + debug_assert(entry->dirty_generation == 0); + // Stamp before clearing CC_CLEAN so a fence that observes the page dirty + // always sees a valid generation. clear_flag is a full barrier. + entry->dirty_generation = + __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); + clockcache_clear_flag(cc, entry_number, CC_CLEAN); + } +} + +/* * Mark an entry clean once its writeback has completed. CC_CLEAN must be set * before CC_WRITEBACK is cleared: the intermediate CC_CLEAN|CC_WRITEBACK state - * is not cleanable, so no thread can start a duplicate writeback in the gap. A - * write lock cannot be held while CC_WRITEBACK is set (see - * clockcache_get_write), so the clean<->dirty transitions never race and no - * lock is needed here. - *-------------------------------------------------------------------------- + * is not cleanable, so no thread can start a duplicate writeback in the gap. */ static void clockcache_dirty_complete_writeback(clockcache *cc, uint32 entry_number) { + clockcache_get_entry(cc, entry_number)->dirty_generation = 0; debug_only uint32 was_clean = clockcache_set_flag(cc, entry_number, CC_CLEAN); debug_assert(!was_clean); @@ -820,6 +841,13 @@ clockcache_ok_to_writeback(clockcache *cc, * status must be: * -- CC_CLEANABLE1_STATUS (= 0) // dirty * -- CC_CLEANABLE2_STATUS (= 0 | CC_ACCESSED) // dirty + * + * Returns FALSE only if the page is genuinely not writeback-able (locked, + * claimed, already in writeback, clean, ...). The CC_ACCESSED bit can flip + * (a reader sets it, the clock hand clears it) between the two + *compare-and- swaps, so we retry as long as the status remains one of the + *cleanable states rather than spuriously failing on a page that stayed + *cleanable. *---------------------------------------------------------------------- */ static inline bool32 @@ -834,19 +862,27 @@ clockcache_try_set_writeback(clockcache *cc, cc->cfg->page_capacity); volatile uint32 *status = &cc->entry[entry_number].status; - if (__sync_bool_compare_and_swap( - status, CC_CLEANABLE1_STATUS, CC_WRITEBACK1_STATUS)) - { - return TRUE; - } - - if (with_access - && __sync_bool_compare_and_swap( - status, CC_CLEANABLE2_STATUS, CC_WRITEBACK2_STATUS)) - { - return TRUE; + while (TRUE) { + uint32 cur = *status; + if (cur == CC_CLEANABLE1_STATUS) { + if (__sync_bool_compare_and_swap( + status, CC_CLEANABLE1_STATUS, CC_WRITEBACK1_STATUS)) + { + return TRUE; + } + } else if (with_access && cur == CC_CLEANABLE2_STATUS) { + if (__sync_bool_compare_and_swap( + status, CC_CLEANABLE2_STATUS, CC_WRITEBACK2_STATUS)) + { + return TRUE; + } + } else { + // Not a cleanable state: the page cannot be written back right now. + return FALSE; + } + // The CAS failed because the status changed under us. If it is still + // cleanable, retry; otherwise the next iteration returns FALSE. } - return FALSE; } typedef struct async_io_state { @@ -1091,42 +1127,72 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) /* *-------------------------------------------------------------------------- - * clockcache_writeback_fence -- - * - * Issue writeback for every dirty, unlocked page and wait for all of it to - * complete, so that a subsequent durable_barrier can make it durable. - * - * A single pass over the cache suffices. Everything reachable from a published - * checkpoint root is copy-on-write and therefore never write-locked while it is - * reachable, so a dirty page that a writer currently holds locked cannot be - * part of this checkpoint and is safely skipped (clockcache_ok_to_writeback - * rejects locked pages). Pages that a concurrent cleaner already has in - * writeback are not re-issued here but are still drained by the final - * io_wait_all. The fence lock serializes checkpoints so that only one thread - * drains the shared I/O contexts at a time. + * clockcache_writeback_dirty -- + * + * Write back every page that was dirty when this call began, and wait for + * those writes (only those) to complete, so a subsequent durable_barrier can + * make them durable. + * + * We take a "cut" by incrementing the dirty generation: pages dirtied before + * now carry a generation <= cutoff and must be drained; pages dirtied + * afterward carry a higher generation and never delay this call. This is what + * lets us avoid io_wait_all, which waits for *global* I/O quiescence -- a + * condition a busy cache (background cleaner writes, async reads) may never + * reach. + * + * A single scan of the entries suffices, because a generation only ever moves + * above the cut (a page must go clean before it can be re-dirtied), so an entry + * that has left the pre-cut set never re-enters it. We first issue writeback + * for every dirty, unlocked page in one bulk pass so the writes pipeline, then + * walk the entries once, waiting per entry for its pre-cut interval to drain. + * + * The bulk pass leaves every pre-cut page that belongs to this checkpoint in + * CC_WRITEBACK: such a page is dirty and, under copy-on-write, never locked or + * claimed, so clockcache_try_set_writeback issues it (or a concurrent cleaner + * already has). The drain therefore only has to wait on pages that are in + * writeback; a pre-cut page that is not in writeback is either already clean, + * or was held by a writer during the bulk pass and so cannot belong to this + * checkpoint -- either way we skip it, which is what keeps this deadlock-free. + * Per-context background reapers complete the issued writes; we also poll our + * own context via clockcache_wait to help things along. + * + * Termination: once a page is in writeback the writer is excluded (see + * clockcache_get_write), so it progresses to clean and, if re-dirtied, moves to + * a generation above the cut. *-------------------------------------------------------------------------- */ platform_status -clockcache_writeback_fence(clockcache *cc) +clockcache_writeback_dirty(clockcache *cc) { - platform_status rc = platform_mutex_lock(&cc->writeback_fence_lock); - platform_assert_status_ok(rc); + uint64 cutoff = + __atomic_fetch_add(&cc->dirty_generation, 1, __ATOMIC_SEQ_CST); + platform_assert(cutoff < UINT64_MAX); - platform_status result = STATUS_OK; + // Bulk-issue writeback for every dirty, unlocked page so the writes pipeline + // rather than draining one batch at a time. for (uint64 batch = 0; batch < cc->cfg->batch_capacity; batch++) { - result = clockcache_batch_start_writeback(cc, batch, TRUE); + platform_status result = clockcache_batch_start_writeback(cc, batch, TRUE); if (!SUCCESS(result)) { - break; + return result; } } - // Drain everything issued above (and any in-flight cleaner writeback) so the - // caller's durable_barrier sees completed writes. - io_wait_all(cc->io); + // Wait for each pre-cut interval's writeback to complete, in a single pass. + for (uint32 entry_no = 0; entry_no < cc->cfg->page_capacity; entry_no++) { + clockcache_entry *entry = clockcache_get_entry(cc, entry_no); + while (TRUE) { + uint64 gen = + __atomic_load_n(&entry->dirty_generation, __ATOMIC_RELAXED); + if (gen == 0 || gen > cutoff + || !clockcache_test_flag(cc, entry_no, CC_WRITEBACK)) + { + break; + } + clockcache_wait(cc); + } + } - platform_status unlock_rc = platform_mutex_unlock(&cc->writeback_fence_lock); - platform_assert_status_ok(unlock_rc); - return result; + return STATUS_OK; } /* @@ -1226,8 +1292,9 @@ clockcache_try_evict(clockcache *cc, uint32 entry_number) /* 6. set status to CC_FREE_STATUS (clears claim and write lock) */ platform_assert(entry->waiters.head == NULL); - entry->type = PAGE_TYPE_INVALID; - entry->status = CC_FREE_STATUS; + entry->type = PAGE_TYPE_INVALID; + entry->dirty_generation = 0; + entry->status = CC_FREE_STATUS; clockcache_log( addr, entry_number, "evict: entry %u addr %lu\n", entry_number, addr); @@ -1366,6 +1433,14 @@ clockcache_get_free_page(clockcache *cc, continue; } + // A page that begins dirty (a fresh allocation) must carry a dirty + // generation, just like a clean->dirty transition. The entry is + // write-locked here, so a concurrent fence skips it regardless. + if (!(status & CC_CLEAN)) { + debug_assert(entry->dirty_generation == 0); + entry->dirty_generation = + __atomic_load_n(&cc->dirty_generation, __ATOMIC_RELAXED); + } entry->status = status; entry->type = type; @@ -1593,8 +1668,9 @@ clockcache_try_page_discard(clockcache *cc, uint64 addr) /* 6. set status to CC_FREE_STATUS (clears claim and write lock) */ platform_assert(entry->waiters.head == NULL); - entry->type = PAGE_TYPE_INVALID; - entry->status = CC_FREE_STATUS; + entry->type = PAGE_TYPE_INVALID; + entry->dirty_generation = 0; + entry->status = CC_FREE_STATUS; /* 7. reset pincount */ clockcache_reset_pin(cc, entry_number); @@ -1711,8 +1787,9 @@ clockcache_get_in_cache(clockcache *cc, // IN static void clockcache_release_unpublished_entry(clockcache_entry *entry) { - entry->page.disk_addr = CC_UNMAPPED_ADDR; - entry->type = PAGE_TYPE_INVALID; + entry->page.disk_addr = CC_UNMAPPED_ADDR; + entry->type = PAGE_TYPE_INVALID; + entry->dirty_generation = 0; platform_assert(entry->waiters.head == NULL); entry->status = CC_FREE_STATUS; } @@ -2256,9 +2333,10 @@ clockcache_lock(clockcache *cc, page_handle *page) entry_number, page->disk_addr); clockcache_get_write(cc, entry_number); - // A write lock marks the page dirty; the CC_WRITEBACK exclusion in - // clockcache_get_write guarantees this cannot race a writeback completion. - clockcache_clear_flag(cc, entry_number, CC_CLEAN); + // A write lock marks the page dirty (and stamps its dirty generation). The + // CC_WRITEBACK exclusion in clockcache_get_write guarantees this cannot race + // a writeback completion. + clockcache_dirty_begin(cc, entry_number); } void @@ -2373,11 +2451,12 @@ clockcache_page_writeback(clockcache *cc, if (!is_blocking) { state = TYPED_MALLOC(PROCESS_PRIVATE_HEAP_ID, state); if (state == NULL) { - platform_error_log("clockcache_page_writeback: async_io_state allocation " - "failed for addr %lu, entry %u, type %u\n", - addr, - entry_number, - type); + platform_error_log( + "clockcache_page_writeback: async_io_state allocation " + "failed for addr %lu, entry %u, type %u\n", + addr, + entry_number, + type); } platform_assert(state); state->cc = cc; @@ -2389,34 +2468,37 @@ clockcache_page_writeback(clockcache *cc, clockcache_write_callback, state); if (!SUCCESS(status)) { - platform_error_log("clockcache_page_writeback: io_async_state_init failed " - "for addr %lu, entry %u, type %u: %s\n", - addr, - entry_number, - type, - platform_status_to_string(status)); + platform_error_log( + "clockcache_page_writeback: io_async_state_init failed " + "for addr %lu, entry %u, type %u: %s\n", + addr, + entry_number, + type, + platform_status_to_string(status)); } platform_assert_status_ok(status); status = io_async_state_append_page(state->iostate, page->data); if (!SUCCESS(status)) { - platform_error_log("clockcache_page_writeback: io_async_state_append_page " - "failed for addr %lu, entry %u, type %u: %s\n", - addr, - entry_number, - type, - platform_status_to_string(status)); + platform_error_log( + "clockcache_page_writeback: io_async_state_append_page " + "failed for addr %lu, entry %u, type %u: %s\n", + addr, + entry_number, + type, + platform_status_to_string(status)); } platform_assert_status_ok(status); io_async_run(state->iostate); } else { status = io_write(cc->io, page->data, clockcache_page_size(cc), addr); if (!SUCCESS(status)) { - platform_error_log("clockcache_page_writeback: io_write failed for addr " - "%lu, entry %u, type %u: %s\n", - addr, - entry_number, - type, - platform_status_to_string(status)); + platform_error_log( + "clockcache_page_writeback: io_write failed for addr " + "%lu, entry %u, type %u: %s\n", + addr, + entry_number, + type, + platform_status_to_string(status)); } platform_assert_status_ok(status); clockcache_log(addr, @@ -3261,10 +3343,10 @@ clockcache_flush_virtual(cache *c) } platform_status -clockcache_writeback_fence_virtual(cache *c) +clockcache_writeback_dirty_virtual(cache *c) { clockcache *cc = (clockcache *)c; - return clockcache_writeback_fence(cc); + return clockcache_writeback_dirty(cc); } platform_status @@ -3407,7 +3489,7 @@ static cache_ops clockcache_ops = { .page_writeback = clockcache_page_writeback_virtual, .extent_writeback = clockcache_extent_writeback_virtual, .flush = clockcache_flush_virtual, - .writeback_fence = clockcache_writeback_fence_virtual, + .writeback_dirty = clockcache_writeback_dirty_virtual, .durable_barrier = clockcache_durable_barrier_virtual, .evict = clockcache_evict_all_virtual, .cleanup = clockcache_wait_virtual, @@ -3501,18 +3583,12 @@ clockcache_init(clockcache *cc, // OUT cc->io = io; cc->heap_id = hid; - platform_status rc = - platform_mutex_init(&cc->writeback_fence_lock, mid, hid); - if (!SUCCESS(rc)) { - platform_error_log( - "clockcache_init: failed to initialize writeback fence lock: %s\n", - platform_status_to_string(rc)); - goto alloc_error; - } - cc->writeback_fence_lock_initialized = TRUE; + // Generation 0 is the sentinel for "clean/free", so dirty stamping starts + // at 1. + cc->dirty_generation = 1; /* lookup maps addrs to entries, entry contains the entries themselves */ - rc = platform_buffer_init(&cc->lookup_bh, + platform_status rc = platform_buffer_init(&cc->lookup_bh, allocator_page_capacity * sizeof(cc->lookup[0])); if (!SUCCESS(rc)) { platform_error_log("clockcache_init: failed to allocate lookup table " @@ -3552,9 +3628,10 @@ clockcache_init(clockcache *cc, // OUT for (i = 0; i < cc->cfg->page_capacity; i++) { cc->entry[i].page.data = cc->data + clockcache_multiply_by_page_size(cc, i); - cc->entry[i].page.disk_addr = CC_UNMAPPED_ADDR; - cc->entry[i].status = CC_FREE_STATUS; - cc->entry[i].type = PAGE_TYPE_INVALID; + cc->entry[i].page.disk_addr = CC_UNMAPPED_ADDR; + cc->entry[i].status = CC_FREE_STATUS; + cc->entry[i].dirty_generation = 0; + cc->entry[i].type = PAGE_TYPE_INVALID; async_wait_queue_init(&cc->entry[i].waiters); } @@ -3636,12 +3713,6 @@ clockcache_deinit(clockcache *cc) // IN/OUT #endif } - if (cc->writeback_fence_lock_initialized) { - rc = platform_mutex_destroy(&cc->writeback_fence_lock); - platform_assert_status_ok(rc); - cc->writeback_fence_lock_initialized = FALSE; - } - if (cc->lookup) { rc = platform_buffer_deinit(&cc->lookup_bh); if (!SUCCESS(rc)) { diff --git a/src/clockcache.h b/src/clockcache.h index 0ef394a4..421f070e 100644 --- a/src/clockcache.h +++ b/src/clockcache.h @@ -73,6 +73,10 @@ typedef uint32 entry_status; // Saved in clockcache_entry->status struct clockcache_entry { page_handle page; volatile entry_status status; + // Generation in which this page's current dirty interval began; 0 when the + // page is clean or free. A writeback fence drains every entry whose + // generation is at or below the cut it took (see clockcache_writeback_dirty). + volatile uint64 dirty_generation; page_type type; async_wait_queue waiters; #ifdef RECORD_ACQUISITION_STACKS @@ -140,10 +144,11 @@ struct clockcache { volatile bool32 *batch_busy; // Convenience pointer for batch_bh uint64 cleaner_gap; - // Serializes checkpoint writeback fences so only one thread drains the - // shared I/O contexts at a time (see clockcache_writeback_fence). - platform_mutex writeback_fence_lock; - bool32 writeback_fence_lock_initialized; + // Monotonic generation counter. A writeback fence atomically increments it + // to take a "cut", then drains every entry stamped with a generation at or + // below that cut. Concurrent fences are safe: each takes a distinct cut and + // waits only on its own I/O context, so no lock is needed. + uint64 dirty_generation; volatile struct { volatile uint32 free_hand; diff --git a/src/core.c b/src/core.c index b91d6f2e..9af1975a 100644 --- a/src/core.c +++ b/src/core.c @@ -59,9 +59,8 @@ _Static_assert(CORE_NUM_MEMTABLES <= MAX_MEMTABLES, static platform_status core_checkpoint_lock_init(core_handle *spl) { - platform_status rc = platform_mutex_init(&spl->checkpoint_lock, - platform_get_module_id(), - spl->heap_id); + platform_status rc = platform_mutex_init( + &spl->checkpoint_lock, platform_get_module_id(), spl->heap_id); if (SUCCESS(rc)) { spl->checkpoint_lock_initialized = TRUE; } @@ -165,17 +164,17 @@ typedef struct ONDISK core_checkpoint_record { * The highest memtable generation incorporated in root_addr. The boolean * keeps the fresh-database case distinct from generation zero. */ - uint64 incorporated_generation; - bool32 has_incorporated_generation; - uint64 root_addr; - uint64 timestamp; - uint64 sequence; - uint64 table_id; - uint32 record_slot; - bool32 checkpointed; - bool32 unmounted; - uint64 magic; - uint64 format_version; + uint64 incorporated_generation; + bool32 has_incorporated_generation; + uint64 root_addr; + uint64 timestamp; + uint64 sequence; + uint64 table_id; + uint32 record_slot; + bool32 checkpointed; + bool32 unmounted; + uint64 magic; + uint64 format_version; checksum128 checksum; } core_checkpoint_record; @@ -216,7 +215,7 @@ core_checkpoint_record_addr_is_valid(core_handle *spl, uint64 addr) } static bool32 -core_checkpoint_directory_is_valid(core_handle *spl, +core_checkpoint_directory_is_valid(core_handle *spl, const core_checkpoint_directory *directory) { if (directory->magic != CORE_CHECKPOINT_DIRECTORY_MAGIC @@ -228,19 +227,20 @@ core_checkpoint_directory_is_valid(core_handle *spl, return FALSE; } - uint64 record0 = directory->record_addr[0]; - uint64 record1 = directory->record_addr[1]; + uint64 record0 = directory->record_addr[0]; + uint64 record1 = directory->record_addr[1]; allocator_config *allocator_cfg = allocator_get_config(spl->al); return core_checkpoint_record_addr_is_valid(spl, record0) && core_checkpoint_record_addr_is_valid(spl, record1) && record0 != record1 - && !allocator_config_pages_share_extent(allocator_cfg, record0, record1); + && !allocator_config_pages_share_extent( + allocator_cfg, record0, record1); } static bool32 -core_checkpoint_record_is_valid(core_handle *spl, +core_checkpoint_record_is_valid(core_handle *spl, const core_checkpoint_record *record, - uint64 record_slot) + uint64 record_slot) { return record->magic == CORE_CHECKPOINT_RECORD_MAGIC && record->format_version == CORE_CHECKPOINT_FORMAT_VERSION @@ -293,7 +293,7 @@ core_initialize_checkpoint_record_page(core_handle *spl, uint64 page_addr) } static platform_status -core_create_checkpoint_directory(core_handle *spl, +core_create_checkpoint_directory(core_handle *spl, core_checkpoint_directory *directory) { uint64 directory_addr; @@ -338,7 +338,7 @@ core_create_checkpoint_directory(core_handle *spl, * bootstrap mapping through the shared backing I/O handle, which the * following durable barrier fdatasyncs with the directory page. */ - rc = cache_writeback_fence(spl->cc); + rc = cache_writeback_dirty(spl->cc); if (!SUCCESS(rc)) { return rc; } @@ -346,7 +346,7 @@ core_create_checkpoint_directory(core_handle *spl, } static platform_status -core_get_checkpoint_directory(core_handle *spl, +core_get_checkpoint_directory(core_handle *spl, core_checkpoint_directory *directory) { uint64 directory_addr; @@ -371,21 +371,19 @@ core_get_checkpoint_directory(core_handle *spl, } static platform_status -core_load_checkpoint_records(core_handle *spl, +core_load_checkpoint_records(core_handle *spl, const core_checkpoint_directory *directory, - core_checkpoint_records *records) + core_checkpoint_records *records) { ZERO_CONTENTS(records); for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { - page_handle *page = cache_get(spl->cc, - directory->record_addr[slot], - TRUE, - PAGE_TYPE_SUPERBLOCK); + page_handle *page = cache_get( + spl->cc, directory->record_addr[slot], TRUE, PAGE_TYPE_SUPERBLOCK); memcpy(&records->record[slot], page->data, sizeof(records->record[slot])); cache_unget(spl->cc, page); - records->valid[slot] = core_checkpoint_record_is_valid( - spl, &records->record[slot], slot); + records->valid[slot] = + core_checkpoint_record_is_valid(spl, &records->record[slot], slot); if (!records->valid[slot]) { continue; } @@ -422,8 +420,7 @@ core_load_checkpoint_records(core_handle *spl, static void core_destroy_checkpoint_record_extent(core_handle *spl, uint64 record_addr) { - refcount ref = - allocator_dec_ref(spl->al, record_addr, PAGE_TYPE_SUPERBLOCK); + refcount ref = allocator_dec_ref(spl->al, record_addr, PAGE_TYPE_SUPERBLOCK); if (ref != AL_NO_REFS) { platform_error_log("core_destroy_checkpoint_record_extent: record extent " "%lu has unexpected refcount %u\n", @@ -516,9 +513,8 @@ core_capture_checkpoint_cut(core_handle *spl, } *has_incorporated_generation = retired_generation != UINT64_MAX; - *incorporated_generation = *has_incorporated_generation - ? retired_generation - : 0; + *incorporated_generation = + *has_incorporated_generation ? retired_generation : 0; return STATUS_OK; } @@ -528,11 +524,11 @@ core_publish_checkpoint_record(core_handle *spl, bool32 is_unmount, bool32 is_create) { - uint64 old_root_addr; - platform_status rc; - trunk_snapshot snapshot; - bool32 has_incorporated_generation; - uint64 incorporated_generation; + uint64 old_root_addr; + platform_status rc; + trunk_snapshot snapshot; + bool32 has_incorporated_generation; + uint64 incorporated_generation; core_checkpoint_directory directory; core_checkpoint_records records; uint64 target_slot; @@ -549,10 +545,8 @@ core_publish_checkpoint_record(core_handle *spl, return rc; } - rc = core_capture_checkpoint_cut(spl, - &snapshot, - &has_incorporated_generation, - &incorporated_generation); + rc = core_capture_checkpoint_cut( + spl, &snapshot, &has_incorporated_generation, &incorporated_generation); if (!SUCCESS(rc)) { goto unlock_checkpoint; } @@ -601,32 +595,30 @@ core_publish_checkpoint_record(core_handle *spl, * is durable, then release only the overwritten owner. The newest record * remains independently live as the torn-write fallback. */ - old_root_addr = records.valid[target_slot] - ? records.record[target_slot].root_addr - : 0; + old_root_addr = + records.valid[target_slot] ? records.record[target_slot].root_addr : 0; ZERO_CONTENTS(&record); record.incorporated_generation = incorporated_generation; record.has_incorporated_generation = has_incorporated_generation; - record.root_addr = snapshot.root_addr; - record.timestamp = platform_get_real_time(); - record.sequence = records.have_newest - ? records.record[records.newest_slot].sequence + 1 - : 1; - record.table_id = spl->id; - record.record_slot = target_slot; - record.checkpointed = is_checkpoint; - record.unmounted = is_unmount; - record.magic = CORE_CHECKPOINT_RECORD_MAGIC; - record.format_version = CORE_CHECKPOINT_FORMAT_VERSION; + record.root_addr = snapshot.root_addr; + record.timestamp = platform_get_real_time(); + record.sequence = records.have_newest + ? records.record[records.newest_slot].sequence + 1 + : 1; + record.table_id = spl->id; + record.record_slot = target_slot; + record.checkpointed = is_checkpoint; + record.unmounted = is_unmount; + record.magic = CORE_CHECKPOINT_RECORD_MAGIC; + record.format_version = CORE_CHECKPOINT_FORMAT_VERSION; record.checksum = core_checkpoint_record_checksum(&record); - core_write_checkpoint_page(spl, - directory.record_addr[target_slot], - &record, - sizeof(record)); - /* The record now owns this reference, even if the barrier reports failure. */ + core_write_checkpoint_page( + spl, directory.record_addr[target_slot], &record, sizeof(record)); + /* The record now owns this reference, even if the barrier reports failure. + */ snapshot.root_addr = 0; rc = cache_durable_barrier(spl->cc); @@ -655,21 +647,21 @@ core_publish_checkpoint_record(core_handle *spl, goto unlock_checkpoint; release_snapshot: - { - platform_status release_rc = - trunk_snapshot_release(&spl->trunk_context, &snapshot); - if (SUCCESS(rc) && !SUCCESS(release_rc)) { - rc = release_rc; - } +{ + platform_status release_rc = + trunk_snapshot_release(&spl->trunk_context, &snapshot); + if (SUCCESS(rc) && !SUCCESS(release_rc)) { + rc = release_rc; } +} unlock_checkpoint: - { - platform_status unlock_rc = platform_mutex_unlock(&spl->checkpoint_lock); - if (SUCCESS(rc) && !SUCCESS(unlock_rc)) { - rc = unlock_rc; - } +{ + platform_status unlock_rc = platform_mutex_unlock(&spl->checkpoint_lock); + if (SUCCESS(rc) && !SUCCESS(unlock_rc)) { + rc = unlock_rc; } +} return rc; } @@ -2313,14 +2305,15 @@ core_mkfs(core_handle *spl, platform_status rc = core_checkpoint_lock_init(spl); if (!SUCCESS(rc)) { - platform_error_log("core_mkfs: checkpoint lock initialization failed: %s\n", - platform_status_to_string(rc)); + platform_error_log( + "core_mkfs: checkpoint lock initialization failed: %s\n", + platform_status_to_string(rc)); return rc; } // set up the memtable context memtable_config *mt_cfg = &spl->cfg.mt_cfg; - rc = memtable_context_init(&spl->mt_ctxt, + rc = memtable_context_init(&spl->mt_ctxt, spl->heap_id, cc, mt_cfg, @@ -2359,8 +2352,9 @@ core_mkfs(core_handle *spl, rc = core_publish_checkpoint_record(spl, FALSE, FALSE, TRUE); if (!SUCCESS(rc)) { - platform_error_log("core_mkfs: core_publish_checkpoint_record failed: %s\n", - platform_status_to_string(rc)); + platform_error_log( + "core_mkfs: core_publish_checkpoint_record failed: %s\n", + platform_status_to_string(rc)); goto deinit_stats; } return STATUS_OK; @@ -2405,8 +2399,9 @@ core_mount(core_handle *spl, platform_status rc = core_checkpoint_lock_init(spl); if (!SUCCESS(rc)) { - platform_error_log("core_mount: checkpoint lock initialization failed: %s\n", - platform_status_to_string(rc)); + platform_error_log( + "core_mount: checkpoint lock initialization failed: %s\n", + platform_status_to_string(rc)); return rc; } @@ -2416,7 +2411,7 @@ core_mount(core_handle *spl, * unmounted record supplies the root. We still validate both records and * choose the newest clean one by sequence rather than wall-clock time. */ - uint64 root_addr = 0; + uint64 root_addr = 0; bool32 has_incorporated_generation = FALSE; uint64 incorporated_generation = 0; core_checkpoint_directory directory; @@ -2436,8 +2431,7 @@ core_mount(core_handle *spl, rc = STATUS_BAD_PARAM; goto deinit_checkpoint_lock; } - const core_checkpoint_record *record = - &records.record[records.newest_slot]; + const core_checkpoint_record *record = &records.record[records.newest_slot]; if (!record->unmounted) { /* * This is an interrupted run. An older clean record is only an A/B @@ -2450,12 +2444,12 @@ core_mount(core_handle *spl, rc = STATUS_INVALID_STATE; goto deinit_checkpoint_lock; } - root_addr = record->root_addr; + root_addr = record->root_addr; has_incorporated_generation = record->has_incorporated_generation; incorporated_generation = record->incorporated_generation; memtable_config *mt_cfg = &spl->cfg.mt_cfg; - rc = memtable_context_init_at_generation( + rc = memtable_context_init_at_generation( &spl->mt_ctxt, spl->heap_id, cc, @@ -2496,8 +2490,9 @@ core_mount(core_handle *spl, rc = core_publish_checkpoint_record(spl, FALSE, FALSE, FALSE); if (!SUCCESS(rc)) { - platform_error_log("core_mount: core_publish_checkpoint_record failed: %s\n", - platform_status_to_string(rc)); + platform_error_log( + "core_mount: core_publish_checkpoint_record failed: %s\n", + platform_status_to_string(rc)); goto deinit_stats; } return STATUS_OK; @@ -2626,8 +2621,9 @@ core_unmount(core_handle *spl) core_quiesce_for_shutdown(spl); rc = core_publish_checkpoint_record(spl, FALSE, TRUE, FALSE); if (!SUCCESS(rc)) { - platform_error_log("core_unmount: failed to publish checkpoint record: %s\n", - platform_status_to_string(rc)); + platform_error_log( + "core_unmount: failed to publish checkpoint record: %s\n", + platform_status_to_string(rc)); } core_teardown_after_shutdown(spl); trunk_context_deinit(&spl->trunk_context); diff --git a/src/shard_log.c b/src/shard_log.c index 205ed3ca..f9096323 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -395,7 +395,7 @@ shard_log_write(log_handle *logh, * In particular, thread_data is otherwise only protected by the * per-thread writer convention, not by a log-wide lock. This function * intentionally does not issue writeback: after establishing that - * exclusion, the caller takes cache_writeback_fence(), followed by a + * exclusion, the caller takes cache_writeback_dirty(), followed by a * durable barrier. */ platform_status diff --git a/src/trunk.c b/src/trunk.c index c95c0c72..8ebbc0b2 100644 --- a/src/trunk.c +++ b/src/trunk.c @@ -6852,7 +6852,7 @@ trunk_context_clone(trunk_context *dst, trunk_context *src) platform_status trunk_make_durable(trunk_context *context) { - platform_status rc = cache_writeback_fence(context->cc); + platform_status rc = cache_writeback_dirty(context->cc); if (!SUCCESS(rc)) { return rc; } diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index f6ce1d75..7e4a06b0 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -113,9 +113,9 @@ cache_test_verify_disk_page(cache *cc, uint64 page_size, uint8 expected) { - clockcache *clock = (clockcache *)cc; - buffer_handle buffer; - platform_status rc = platform_buffer_init(&buffer, page_size); + clockcache *clock = (clockcache *)cc; + buffer_handle buffer; + platform_status rc = platform_buffer_init(&buffer, page_size); if (!SUCCESS(rc)) { return rc; } @@ -180,8 +180,8 @@ cache_test_zero_disk_page(io_handle *io, uint64 addr, uint64 page_size) */ static platform_status test_rc_allocator_recovery_bootstrap(allocator_config *cfg, - io_handle *io, - platform_heap_id hid) + io_handle *io, + platform_heap_id hid) { const allocator_root_id root_id = 1; uint64 refcount_buffer_size = @@ -192,28 +192,28 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, uint64 reserved_extent_count = 1 + refcount_extent_count + 2; uint64 old_clean_state_addr = (1 + refcount_extent_count + 1) * cfg->io_cfg->extent_size; - uint64 super_addr = 0; - uint64 stale_extent_addr = 0; - platform_status rc = STATUS_OK; - rc_allocator original, recovery, remounted; - bool32 original_live = FALSE; - bool32 recovery_live = FALSE; - bool32 remounted_live = FALSE; + uint64 super_addr = 0; + uint64 stale_extent_addr = 0; + platform_status rc = STATUS_OK; + rc_allocator original, recovery, remounted; + bool32 original_live = FALSE; + bool32 recovery_live = FALSE; + bool32 remounted_live = FALSE; ZERO_CONTENTS(&original); ZERO_CONTENTS(&recovery); ZERO_CONTENTS(&remounted); - platform_default_log("cache_test: allocator recovery bootstrap test started\n"); + platform_default_log( + "cache_test: allocator recovery bootstrap test started\n"); - rc = rc_allocator_init( - &original, cfg, io, hid, platform_get_module_id()); + rc = rc_allocator_init(&original, cfg, io, hid, platform_get_module_id()); if (!SUCCESS(rc)) { goto cleanup; } original_live = TRUE; - rc = allocator_alloc_super_addr( - (allocator *)&original, root_id, &super_addr); + rc = + allocator_alloc_super_addr((allocator *)&original, root_id, &super_addr); if (!SUCCESS(rc)) { goto cleanup; } @@ -239,8 +239,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, goto cleanup; } - rc = rc_allocator_mount( - &remounted, cfg, io, hid, platform_get_module_id()); + rc = rc_allocator_mount(&remounted, cfg, io, hid, platform_get_module_id()); if (!SUCCESS(rc)) { platform_error_log("cache_test: normal mount did not fall back to its " "surviving clean-state record\n"); @@ -260,8 +259,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, rc_allocator_deinit(&remounted); remounted_live = FALSE; - rc = rc_allocator_mount( - &remounted, cfg, io, hid, platform_get_module_id()); + rc = rc_allocator_mount(&remounted, cfg, io, hid, platform_get_module_id()); if (rc.r != STATUS_INVALID_STATE.r) { platform_error_log("cache_test: normal mount trusted an allocator after " "a simulated mounted crash\n"); @@ -278,7 +276,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, recovery_live = TRUE; uint64 mounted_super_addr = 0; - rc = allocator_get_super_addr( + rc = allocator_get_super_addr( (allocator *)&recovery, root_id, &mounted_super_addr); if (!SUCCESS(rc) || mounted_super_addr != super_addr) { platform_error_log("cache_test: recovery lost a persisted superblock " @@ -294,8 +292,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, rc = STATUS_TEST_FAILED; goto cleanup; } - for (uint64 extent_no = 0; extent_no < reserved_extent_count; extent_no++) - { + for (uint64 extent_no = 0; extent_no < reserved_extent_count; extent_no++) { uint64 extent_addr = extent_no * cfg->io_cfg->extent_size; if (allocator_get_refcount((allocator *)&recovery, extent_addr) != AL_ONE_REF) @@ -333,8 +330,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, rc_allocator_abort_recovery(&recovery); recovery_live = FALSE; - rc = rc_allocator_mount( - &remounted, cfg, io, hid, platform_get_module_id()); + rc = rc_allocator_mount(&remounted, cfg, io, hid, platform_get_module_id()); if (rc.r != STATUS_INVALID_STATE.r) { platform_error_log("cache_test: normal mount trusted an unclean " "allocator map\n"); @@ -368,8 +364,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, rc_allocator_unmount(&recovery); recovery_live = FALSE; - rc = rc_allocator_mount( - &remounted, cfg, io, hid, platform_get_module_id()); + rc = rc_allocator_mount(&remounted, cfg, io, hid, platform_get_module_id()); if (!SUCCESS(rc)) { goto cleanup; } @@ -415,7 +410,7 @@ static void cache_test_writeback_fence_thread(void *arg) { cache_fence_test_context *ctxt = (cache_fence_test_context *)arg; - ctxt->status = cache_writeback_fence(ctxt->cc); + ctxt->status = cache_writeback_dirty(ctxt->cc); __atomic_store_n(&ctxt->finished, TRUE, __ATOMIC_RELEASE); } @@ -431,19 +426,18 @@ test_cache_writeback_fence(cache *cc, clockcache_config *cfg, platform_heap_id hid) { - platform_status rc = STATUS_OK; - uint64 pages_per_extent = - cache_config_pages_per_extent(&cfg->super); - uint64 *addr_arr = NULL; - page_handle *old_page = NULL; - page_handle *new_page = NULL; - bool32 new_claimed = FALSE, new_locked = FALSE; - bool32 fence_thread_started = FALSE; - const uint8 old_baseline = 0x11, old_value = 0x22; - const uint8 new_baseline = 0x33, new_value = 0x44; + platform_status rc = STATUS_OK; + uint64 pages_per_extent = cache_config_pages_per_extent(&cfg->super); + uint64 *addr_arr = NULL; + page_handle *old_page = NULL; + page_handle *new_page = NULL; + bool32 new_claimed = FALSE, new_locked = FALSE; + bool32 fence_thread_started = FALSE; + const uint8 old_baseline = 0x11, old_value = 0x22; + const uint8 new_baseline = 0x33, new_value = 0x44; cache_fence_test_context fence_ctxt = { .cc = cc, .finished = FALSE, .status = STATUS_OK}; - platform_thread fence_thread; + platform_thread fence_thread; platform_default_log("cache_test: writeback fence test started\n"); platform_assert(cfg->page_capacity >= 2 * pages_per_extent); @@ -559,10 +553,8 @@ test_cache_writeback_fence(cache *cc, } /* The dirty, unlocked page was flushed by the fence. */ if (SUCCESS(rc)) { - rc = cache_test_verify_disk_page(cc, - addr_arr[0], - cache_config_page_size(&cfg->super), - old_value); + rc = cache_test_verify_disk_page( + cc, addr_arr[0], cache_config_page_size(&cfg->super), old_value); } /* The write-locked page was skipped, so its baseline is still on disk. */ if (SUCCESS(rc)) { @@ -604,17 +596,16 @@ test_cache_writeback_fence_claimed_page(cache *cc, clockcache_config *cfg, platform_heap_id hid) { - platform_status rc = STATUS_OK; - uint64 pages_per_extent = - cache_config_pages_per_extent(&cfg->super); - uint64 *addr_arr = NULL; - page_handle *page = NULL; - bool32 claimed = FALSE, locked = FALSE; - bool32 fence_thread_started = FALSE; - const uint8 baseline = 0x55, final_value = 0x66; + platform_status rc = STATUS_OK; + uint64 pages_per_extent = cache_config_pages_per_extent(&cfg->super); + uint64 *addr_arr = NULL; + page_handle *page = NULL; + bool32 claimed = FALSE, locked = FALSE; + bool32 fence_thread_started = FALSE; + const uint8 baseline = 0x55, final_value = 0x66; cache_fence_test_context fence_ctxt = { .cc = cc, .finished = FALSE, .status = STATUS_OK}; - platform_thread fence_thread; + platform_thread fence_thread; platform_default_log("cache_test: claimed-page fence test started\n"); @@ -671,7 +662,7 @@ test_cache_writeback_fence_claimed_page(cache *cc, } platform_status join_rc = platform_thread_join(&fence_thread); - fence_thread_started = FALSE; + fence_thread_started = FALSE; if (!SUCCESS(join_rc)) { rc = join_rc; goto cleanup; @@ -688,13 +679,14 @@ test_cache_writeback_fence_claimed_page(cache *cc, goto cleanup; } - /* Drop the claim; the page is now cleanable and a second fence flushes it. */ + /* Drop the claim; the page is now cleanable and a second fence flushes it. + */ cache_unclaim(cc, page); claimed = FALSE; cache_unget(cc, page); page = NULL; - rc = cache_writeback_fence(cc); + rc = cache_writeback_dirty(cc); if (!SUCCESS(rc)) { goto cleanup; } @@ -732,8 +724,8 @@ test_cache_writeback_fence_claimed_page(cache *cc, if (addr_arr != NULL) { cache_flush(cc); - allocator *al = cache_get_allocator(cc); - refcount ref = allocator_dec_ref(al, addr_arr[0], PAGE_TYPE_MISC); + allocator *al = cache_get_allocator(cc); + refcount ref = allocator_dec_ref(al, addr_arr[0], PAGE_TYPE_MISC); platform_assert(ref == AL_NO_REFS); cache_extent_discard(cc, addr_arr[0], PAGE_TYPE_MISC); ref = allocator_dec_ref(al, addr_arr[0], PAGE_TYPE_MISC); @@ -749,6 +741,166 @@ test_cache_writeback_fence_claimed_page(cache *cc, return rc; } +typedef struct { + cache *cc; + uint64 *addrs; + uint64 num_addrs; + uint64 page_size; + volatile bool32 stop; +} cache_hammer_context; + +/* + * Continuously dirty and asynchronously write back a rotating set of pages, so + * the I/O subsystem never reaches global quiescence. + */ +static void +cache_test_hammer_thread(void *arg) +{ + cache_hammer_context *ctxt = (cache_hammer_context *)arg; + uint64 i = 0; + while (!__atomic_load_n(&ctxt->stop, __ATOMIC_ACQUIRE)) { + uint64 addr = ctxt->addrs[i % ctxt->num_addrs]; + page_handle *page = cache_get(ctxt->cc, addr, TRUE, PAGE_TYPE_MISC); + if (cache_try_claim(ctxt->cc, page)) { + cache_lock(ctxt->cc, page); + memset(page->data, (uint8)i, ctxt->page_size); + cache_unlock(ctxt->cc, page); + cache_unclaim(ctxt->cc, page); + cache_page_writeback(ctxt->cc, page, FALSE, PAGE_TYPE_MISC); + } + cache_unget(ctxt->cc, page); + i++; + } +} + +/* + * A writeback fence must drain only the pages dirty at its cut and then return, + * even while another thread keeps dirtying and writing back other pages. The + * previous io_wait_all implementation waited for *global* I/O quiescence, which + * this workload never reaches, so it would hang here. + */ +static platform_status +test_cache_writeback_fence_liveness(cache *cc, + clockcache_config *cfg, + platform_heap_id hid) +{ + platform_status rc = STATUS_OK; + uint64 page_size = cache_config_page_size(&cfg->super); + uint64 pages_per_extent = + cache_config_pages_per_extent(&cfg->super); + uint64 *addr_arr = NULL; + const uint8 baseline = 0x11, target_value = 0x77; + bool32 hammer_started = FALSE, fence_started = FALSE; + cache_hammer_context hammer_ctxt = {.cc = cc, .stop = FALSE}; + cache_fence_test_context fence_ctxt = { + .cc = cc, .finished = FALSE, .status = STATUS_OK}; + platform_thread hammer_thread, fence_thread; + + platform_default_log("cache_test: writeback fence liveness test started\n"); + platform_assert(cfg->page_capacity >= 2 * pages_per_extent); + + addr_arr = TYPED_ARRAY_MALLOC(hid, addr_arr, 2 * pages_per_extent); + if (addr_arr == NULL) { + rc = STATUS_NO_MEMORY; + goto cleanup; + } + rc = cache_test_alloc_extents(cc, cfg, addr_arr, 2); + if (!SUCCESS(rc)) { + platform_free(hid, addr_arr); + addr_arr = NULL; + goto cleanup; + } + rc = cache_test_fill_page(cc, addr_arr[0], page_size, baseline); + if (!SUCCESS(rc)) { + goto cleanup; + } + cache_flush(cc); + + /* Dirty the target page (unlocked); the fence must flush it. */ + rc = cache_test_fill_page(cc, addr_arr[0], page_size, target_value); + if (!SUCCESS(rc)) { + goto cleanup; + } + + /* Hammer the second extent's pages to keep I/O perpetually in flight. */ + hammer_ctxt.addrs = &addr_arr[pages_per_extent]; + hammer_ctxt.num_addrs = pages_per_extent; + hammer_ctxt.page_size = page_size; + rc = platform_thread_create( + &hammer_thread, FALSE, cache_test_hammer_thread, &hammer_ctxt, hid); + if (!SUCCESS(rc)) { + goto cleanup; + } + hammer_started = TRUE; + + rc = platform_thread_create( + &fence_thread, FALSE, cache_test_writeback_fence_thread, &fence_ctxt, hid); + if (!SUCCESS(rc)) { + goto cleanup; + } + fence_started = TRUE; + + /* The fence must return despite the ongoing hammer activity. */ + timestamp wait_start = platform_get_timestamp(); + while (!__atomic_load_n(&fence_ctxt.finished, __ATOMIC_ACQUIRE)) { + if (platform_timestamp_elapsed(wait_start) > SEC_TO_NSEC(10)) { + platform_error_log( + "cache_test: fence did not terminate under concurrent writes\n"); + rc = STATUS_TIMEDOUT; + break; + } + platform_sleep_ns(1000); + } + +cleanup: + /* Stop the hammer first so I/O can quiesce even if the fence is stuck. */ + if (hammer_started) { + __atomic_store_n(&hammer_ctxt.stop, TRUE, __ATOMIC_RELEASE); + platform_status jrc = platform_thread_join(&hammer_thread); + if (!SUCCESS(jrc) && SUCCESS(rc)) { + rc = jrc; + } + } + if (fence_started) { + platform_status jrc = platform_thread_join(&fence_thread); + if (!SUCCESS(jrc) && SUCCESS(rc)) { + rc = jrc; + } + if (!SUCCESS(fence_ctxt.status) && SUCCESS(rc)) { + rc = fence_ctxt.status; + } + } + + /* The target page, dirty at the cut, must have been made durable. */ + if (SUCCESS(rc)) { + rc = cache_durable_barrier(cc); + } + if (SUCCESS(rc)) { + rc = cache_test_verify_disk_page(cc, addr_arr[0], page_size, target_value); + } + + if (addr_arr != NULL) { + cache_flush(cc); + for (uint32 i = 0; i < 2; i++) { + uint64 addr = addr_arr[i * pages_per_extent]; + allocator *al = cache_get_allocator(cc); + refcount ref = allocator_dec_ref(al, addr, PAGE_TYPE_MISC); + platform_assert(ref == AL_NO_REFS); + cache_extent_discard(cc, addr, PAGE_TYPE_MISC); + ref = allocator_dec_ref(al, addr, PAGE_TYPE_MISC); + platform_assert(ref == AL_FREE); + } + platform_free(hid, addr_arr); + } + + if (SUCCESS(rc)) { + platform_default_log("cache_test: writeback fence liveness test passed\n"); + } else { + platform_default_log("cache_test: writeback fence liveness test failed\n"); + } + return rc; +} + platform_status test_cache_basic(cache *cc, clockcache_config *cfg, platform_heap_id hid) { @@ -767,6 +919,11 @@ test_cache_basic(cache *cc, clockcache_config *cfg, platform_heap_id hid) goto exit; } + rc = test_cache_writeback_fence_liveness(cc, cfg, hid); + if (!SUCCESS(rc)) { + goto exit; + } + /* allocate twice as many pages as the cache capacity */ uint64 pages_per_extent = cache_config_pages_per_extent(&cfg->super); uint32 extent_capacity = cfg->page_capacity / pages_per_extent; @@ -1644,8 +1801,8 @@ cache_test(int argc, char *argv[]) goto cleanup; } - rc = test_rc_allocator_recovery_bootstrap( - &system_cfg.allocator_cfg, io, hid); + rc = + test_rc_allocator_recovery_bootstrap(&system_cfg.allocator_cfg, io, hid); if (!SUCCESS(rc)) { goto destroy_iohandle; } diff --git a/tests/functional/log_test.c b/tests/functional/log_test.c index 6a16d05f..fa17d8b4 100644 --- a/tests/functional/log_test.c +++ b/tests/functional/log_test.c @@ -71,19 +71,18 @@ test_log_crash(clockcache *cc, * (memtable_generation, leaf_generation) order below. */ uint64 memtable_generation = 1 - i / LOG_TEST_LEAVES_PER_MEMTABLE; - uint64 leaf_generation = LOG_TEST_LEAVES_PER_MEMTABLE - 1 - - i % LOG_TEST_LEAVES_PER_MEMTABLE; + uint64 leaf_generation = + LOG_TEST_LEAVES_PER_MEMTABLE - 1 - i % LOG_TEST_LEAVES_PER_MEMTABLE; entry_num = memtable_generation * LOG_TEST_LEAVES_PER_MEMTABLE + leaf_generation; } - key skey = - test_key(&keybuffer, - TEST_RANDOM, - entry_num, - 0, - 0, - 1 + (entry_num % key_size), - 0); + key skey = test_key(&keybuffer, + TEST_RANDOM, + entry_num, + 0, + 0, + 1 + (entry_num % key_size), + 0); generate_test_message(gen, entry_num, &msg); int log_rc = log_write(logh, skey, @@ -95,7 +94,7 @@ test_log_crash(clockcache *cc, rc = log_seal(logh); platform_assert_status_ok(rc); - rc = cache_writeback_fence((cache *)cc); + rc = cache_writeback_dirty((cache *)cc); platform_assert_status_ok(rc); rc = cache_durable_barrier((cache *)cc); platform_assert_status_ok(rc); @@ -121,8 +120,7 @@ test_log_crash(clockcache *cc, uint64 leaf_generation; shard_log_iterator_curr_generations( &itor, &memtable_generation, &leaf_generation); - platform_assert( - memtable_generation == i / LOG_TEST_LEAVES_PER_MEMTABLE); + platform_assert(memtable_generation == i / LOG_TEST_LEAVES_PER_MEMTABLE); platform_assert(leaf_generation == i % LOG_TEST_LEAVES_PER_MEMTABLE); if (data_key_compare(cfg->data_cfg, skey, returned_key) || message_lex_cmp(mmessage, returned_message)) @@ -166,7 +164,7 @@ test_log_write_range(log_handle *logh, for (uint64 i = 0; i < num_entries; i++) { uint64 entry_num = first_entry + i; - key skey = test_key(&keybuffer, + key skey = test_key(&keybuffer, TEST_RANDOM, entry_num, 0, @@ -174,11 +172,8 @@ test_log_write_range(log_handle *logh, 1 + (entry_num % key_size), 0); generate_test_message(gen, entry_num, &msg); - int log_rc = log_write(logh, - skey, - merge_accumulator_to_message(&msg), - entry_num, - 0); + int log_rc = log_write( + logh, skey, merge_accumulator_to_message(&msg), entry_num, 0); platform_assert(log_rc == 0); } @@ -228,9 +223,9 @@ test_log_verify_segment(cache *cc, platform_assert(memtable_generation == entry_num); platform_assert(leaf_generation == 0); platform_assert(data_key_compare(cfg->data_cfg, skey, returned_key) == 0); - platform_assert(message_lex_cmp( - merge_accumulator_to_message(&msg), returned_message) - == 0); + platform_assert( + message_lex_cmp(merge_accumulator_to_message(&msg), returned_message) + == 0); rc = iterator_next(itorh); platform_assert_status_ok(rc); } @@ -257,8 +252,8 @@ test_log_rotate(clockcache *cc, test_message_generator *gen, uint64 key_size) { - const uint64 old_first = 1000, old_count = 16; - const uint64 new_first = 2000, new_count = 16; + const uint64 old_first = 1000, old_count = 16; + const uint64 new_first = 2000, new_count = 16; log_segment_info sealed, fresh; platform_status rc = shard_log_init(log, (cache *)cc, cfg); @@ -275,7 +270,7 @@ test_log_rotate(clockcache *cc, platform_assert(sealed.meta_addr != fresh.meta_addr); platform_assert(sealed.magic != fresh.magic); - rc = cache_writeback_fence((cache *)cc); + rc = cache_writeback_dirty((cache *)cc); platform_assert_status_ok(rc); rc = cache_durable_barrier((cache *)cc); platform_assert_status_ok(rc); @@ -284,20 +279,14 @@ test_log_rotate(clockcache *cc, rc = clockcache_init( cc, cache_cfg, io, al, "rotated-old", hid, platform_get_module_id()); platform_assert_status_ok(rc); - test_log_verify_segment((cache *)cc, - cfg, - &sealed, - gen, - hid, - key_size, - old_first, - old_count); + test_log_verify_segment( + (cache *)cc, cfg, &sealed, gen, hid, key_size, old_first, old_count); test_log_write_range( (log_handle *)log, gen, hid, key_size, new_first, new_count); rc = log_seal((log_handle *)log); platform_assert_status_ok(rc); - rc = cache_writeback_fence((cache *)cc); + rc = cache_writeback_dirty((cache *)cc); platform_assert_status_ok(rc); rc = cache_durable_barrier((cache *)cc); platform_assert_status_ok(rc); @@ -306,22 +295,10 @@ test_log_rotate(clockcache *cc, rc = clockcache_init( cc, cache_cfg, io, al, "rotated-new", hid, platform_get_module_id()); platform_assert_status_ok(rc); - test_log_verify_segment((cache *)cc, - cfg, - &sealed, - gen, - hid, - key_size, - old_first, - old_count); - test_log_verify_segment((cache *)cc, - cfg, - &fresh, - gen, - hid, - key_size, - new_first, - new_count); + test_log_verify_segment( + (cache *)cc, cfg, &sealed, gen, hid, key_size, old_first, old_count); + test_log_verify_segment( + (cache *)cc, cfg, &fresh, gen, hid, key_size, new_first, new_count); shard_log_segment_discard((cache *)cc, &sealed); shard_log_zap(log); @@ -353,9 +330,8 @@ test_log_large_message(cache *cc, merge_accumulator_set_class(&msg, MESSAGE_TYPE_INSERT); memset(merge_accumulator_data(&msg), 'L', value_len); - int log_rc = - log_write( - (log_handle *)log, skey, merge_accumulator_to_message(&msg), 0, 0); + int log_rc = log_write( + (log_handle *)log, skey, merge_accumulator_to_message(&msg), 0, 0); platform_assert(log_rc == 0); merge_accumulator filler; @@ -366,19 +342,18 @@ test_log_large_message(cache *cc, memset( merge_accumulator_data(&filler), 'f', merge_accumulator_length(&filler)); for (uint64 i = 1; i < 16; i++) { - log_rc = log_write( - (log_handle *)log, - skey, - merge_accumulator_to_message(&filler), - i / 4, - i % 4); + log_rc = log_write((log_handle *)log, + skey, + merge_accumulator_to_message(&filler), + i / 4, + i % 4); platform_assert(log_rc == 0); } merge_accumulator_deinit(&filler); rc = log_seal((log_handle *)log); platform_assert_status_ok(rc); - rc = cache_writeback_fence(cc); + rc = cache_writeback_dirty(cc); platform_assert_status_ok(rc); rc = cache_durable_barrier(cc); platform_assert_status_ok(rc); @@ -433,11 +408,8 @@ test_log_thread(void *arg) for (i = thread_id * num_entries; i < (thread_id + 1) * num_entries; i++) { key skey = test_key(&keybuf, TEST_RANDOM, i, 0, 0, key_size, 0); generate_test_message(gen, i, &msg); - int log_rc = log_write(logh, - skey, - merge_accumulator_to_message(&msg), - i / 1024, - i % 1024); + int log_rc = log_write( + logh, skey, merge_accumulator_to_message(&msg), i / 1024, i % 1024); platform_assert(log_rc == 0); } From f5ed61502fd47e96dcfaeb683ac23a3d435872e5 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 17 Jul 2026 23:05:42 -0700 Subject: [PATCH 06/64] type decl cleanup Signed-off-by: Rob Johnson --- src/cache.h | 84 ++++++++++++++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/src/cache.h b/src/cache.h index 8e37827b..598df0f5 100644 --- a/src/cache.h +++ b/src/cache.h @@ -94,19 +94,6 @@ cache_config_extent_page(const cache_config *cfg, uint64 extent_addr, uint64 i) return extent_addr + i * cache_config_page_size(cfg); } -typedef void (*cache_generic_fn)(cache *cc); -typedef uint64 (*cache_generic_uint64_fn)(cache *cc); -typedef void (*page_generic_fn)(cache *cc, page_handle *page); -typedef platform_status (*cache_durable_barrier_fn)(cache *cc); -typedef platform_status (*cache_writeback_dirty_fn)(cache *cc); - -typedef page_handle *(*page_alloc_fn)(cache *cc, uint64 addr, page_type type); -typedef void (*extent_discard_fn)(cache *cc, uint64 addr, page_type type); -typedef page_handle *(*page_get_fn)(cache *cc, - uint64 addr, - bool32 blocking, - page_type type); - #define PAGE_GET_ASYNC_STATE_BUFFER_SIZE (2048) typedef union page_get_async_state_payload { uint8 bytes[PAGE_GET_ASYNC_STATE_BUFFER_SIZE]; @@ -120,6 +107,11 @@ typedef struct page_get_async_state_buffer { page_get_async_state_payload payload; } page_get_async_state_buffer; +typedef void (*cache_generic_void_fn)(cache *cc); +typedef uint64 (*cache_generic_uint64_fn)(cache *cc); +typedef platform_status (*cache_generic_status_fn)(cache *cc); +typedef void (*page_generic_fn)(cache *cc, page_handle *page); + typedef void (*page_get_async_state_init_fn)(void *payload, cache *cc, uint64 addr, @@ -129,6 +121,12 @@ typedef void (*page_get_async_state_init_fn)(void *payload, typedef async_status (*page_get_async_fn)(void *payload); typedef page_handle *(*page_get_async_state_result_fn)(void *payload); +typedef page_handle *(*page_alloc_fn)(cache *cc, uint64 addr, page_type type); +typedef void (*extent_discard_fn)(cache *cc, uint64 addr, page_type type); +typedef page_handle *(*page_get_fn)(cache *cc, + uint64 addr, + bool32 blocking, + page_type type); typedef bool32 (*page_try_claim_fn)(cache *cc, page_handle *page); typedef void (*page_writeback_fn)(cache *cc, page_handle *page, @@ -165,36 +163,36 @@ typedef struct cache_ops { page_get_async_fn page_get_async; page_get_async_state_result_fn page_get_async_result; - page_generic_fn page_unget; - page_try_claim_fn page_try_claim; - page_generic_fn page_unclaim; - page_generic_fn page_lock; - page_generic_fn page_unlock; - page_prefetch_fn page_prefetch; - page_prefetch_fn page_prefetch_page; - page_generic_fn page_pin; - page_generic_fn page_unpin; - page_writeback_fn page_writeback; - extent_writeback_fn extent_writeback; - cache_generic_fn flush; - cache_writeback_dirty_fn writeback_dirty; - cache_durable_barrier_fn durable_barrier; - evict_fn evict; - cache_generic_fn cleanup; - page_addr_pred_fn in_use; - page_addr_fn assert_ungot; - cache_generic_fn assert_free; - validate_page_fn validate_page; - cache_present_fn cache_present; - cache_print_fn print; - cache_print_fn print_stats; - io_stats_fn io_stats; - cache_generic_fn reset_stats; - count_dirty_fn count_dirty; - page_get_read_ref_fn page_get_read_ref; - enable_sync_get_fn enable_sync_get; - get_allocator_fn get_allocator; - cache_config_fn get_config; + page_generic_fn page_unget; + page_try_claim_fn page_try_claim; + page_generic_fn page_unclaim; + page_generic_fn page_lock; + page_generic_fn page_unlock; + page_prefetch_fn page_prefetch; + page_prefetch_fn page_prefetch_page; + page_generic_fn page_pin; + page_generic_fn page_unpin; + page_writeback_fn page_writeback; + extent_writeback_fn extent_writeback; + cache_generic_void_fn flush; + cache_generic_status_fn writeback_dirty; + cache_generic_status_fn durable_barrier; + evict_fn evict; + cache_generic_void_fn cleanup; + page_addr_pred_fn in_use; + page_addr_fn assert_ungot; + cache_generic_void_fn assert_free; + validate_page_fn validate_page; + cache_present_fn cache_present; + cache_print_fn print; + cache_print_fn print_stats; + io_stats_fn io_stats; + cache_generic_void_fn reset_stats; + count_dirty_fn count_dirty; + page_get_read_ref_fn page_get_read_ref; + enable_sync_get_fn enable_sync_get; + get_allocator_fn get_allocator; + cache_config_fn get_config; } cache_ops; // To sub-class cache, make a cache your first field; From 68dbb43c8414bf9140926bae784e639294ebbf79 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 17 Jul 2026 23:08:14 -0700 Subject: [PATCH 07/64] clockcache.h cleanup Signed-off-by: Rob Johnson --- src/clockcache.h | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/clockcache.h b/src/clockcache.h index 421f070e..cb5929fd 100644 --- a/src/clockcache.h +++ b/src/clockcache.h @@ -10,7 +10,6 @@ #pragma once #include "platform_buffer.h" -#include "platform_mutex.h" #include "platform_threads.h" #include "allocator.h" #include "cache.h" @@ -74,11 +73,12 @@ struct clockcache_entry { page_handle page; volatile entry_status status; // Generation in which this page's current dirty interval began; 0 when the - // page is clean or free. A writeback fence drains every entry whose - // generation is at or below the cut it took (see clockcache_writeback_dirty). - volatile uint64 dirty_generation; - page_type type; - async_wait_queue waiters; + // page is clean or free. A writeback_dirty() drains every entry whose + // generation is at or below the cut it took (see + // clockcache_writeback_dirty). + volatile uint64 dirty_generation; + page_type type; + async_wait_queue waiters; #ifdef RECORD_ACQUISITION_STACKS int next_history_record; history_record history[NUM_HISTORY_RECORDS]; @@ -144,10 +144,9 @@ struct clockcache { volatile bool32 *batch_busy; // Convenience pointer for batch_bh uint64 cleaner_gap; - // Monotonic generation counter. A writeback fence atomically increments it + // Monotonic generation counter. A writeback_dirty() atomically increments it // to take a "cut", then drains every entry stamped with a generation at or - // below that cut. Concurrent fences are safe: each takes a distinct cut and - // waits only on its own I/O context, so no lock is needed. + // below that cut. uint64 dirty_generation; volatile struct { From a1fc2ea48c1864f373dd6efbe6a826f18bab0df4 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 17 Jul 2026 23:23:48 -0700 Subject: [PATCH 08/64] cleanups Signed-off-by: Rob Johnson --- src/clockcache.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/clockcache.c b/src/clockcache.c index cfa48e34..12ed75c0 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -845,9 +845,9 @@ clockcache_ok_to_writeback(clockcache *cc, * Returns FALSE only if the page is genuinely not writeback-able (locked, * claimed, already in writeback, clean, ...). The CC_ACCESSED bit can flip * (a reader sets it, the clock hand clears it) between the two - *compare-and- swaps, so we retry as long as the status remains one of the - *cleanable states rather than spuriously failing on a page that stayed - *cleanable. + * compare-and- swaps, so we retry as long as the status remains one of the + * cleanable states rather than spuriously failing on a page that stayed + * cleanable. *---------------------------------------------------------------------- */ static inline bool32 @@ -1171,7 +1171,8 @@ clockcache_writeback_dirty(clockcache *cc) // Bulk-issue writeback for every dirty, unlocked page so the writes pipeline // rather than draining one batch at a time. for (uint64 batch = 0; batch < cc->cfg->batch_capacity; batch++) { - platform_status result = clockcache_batch_start_writeback(cc, batch, TRUE); + platform_status result = + clockcache_batch_start_writeback(cc, batch, TRUE); if (!SUCCESS(result)) { return result; } @@ -3588,8 +3589,8 @@ clockcache_init(clockcache *cc, // OUT cc->dirty_generation = 1; /* lookup maps addrs to entries, entry contains the entries themselves */ - platform_status rc = platform_buffer_init(&cc->lookup_bh, - allocator_page_capacity * sizeof(cc->lookup[0])); + platform_status rc = platform_buffer_init( + &cc->lookup_bh, allocator_page_capacity * sizeof(cc->lookup[0])); if (!SUCCESS(rc)) { platform_error_log("clockcache_init: failed to allocate lookup table " "(%lu bytes): %s\n", From b4a055229aea0a51329d45dfba533aabfc2751d1 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sat, 18 Jul 2026 00:01:40 -0700 Subject: [PATCH 09/64] clockcache.c assert on async io contract violation Signed-off-by: Rob Johnson --- src/clockcache.c | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/src/clockcache.c b/src/clockcache.c index 12ed75c0..f006d222 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -1102,21 +1102,18 @@ clockcache_batch_start_writeback(clockcache *cc, uint64 batch, bool32 is_urgent) cc->stats[tid].writes_issued++; } - if (io_async_run(state->iostate) == ASYNC_STATUS_DONE) { - rc = io_async_state_get_result(state->iostate); - if (SUCCESS(rc)) { - platform_error_log( - "clockcache_batch_start_writeback: async write for addr " - "%lu completed without invoking its callback\n", - first_addr); - rc = STATUS_IO_ERROR; - } - clockcache_abort_writeback_range(cc, first_addr, end_addr); - io_async_state_deinit(state->iostate); - platform_free(PROCESS_PRIVATE_HEAP_ID, state); - result = rc; - goto close_log; - } + // The IO layer must run writeback asynchronously and complete it via + // clockcache_write_callback (which does the dirty->clean bookkeeping). + // A synchronous completion would skip that callback and strand the + // pages in CC_WRITEBACK, so a broken contract is a fatal correctness + // error. + async_status arc = io_async_run(state->iostate); + platform_assert( + arc == ASYNC_STATUS_RUNNING, + "clockcache_batch_start_writeback: async writeback for addr %lu " + "completed synchronously; the IO layer must complete it via the " + "write callback", + first_addr); } } From b45f857ec6be780ce603419e6131d42f0afc8342 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sat, 18 Jul 2026 00:12:05 -0700 Subject: [PATCH 10/64] clockcache.c cleanup control flow Signed-off-by: Rob Johnson --- src/clockcache.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/clockcache.c b/src/clockcache.c index f006d222..bd59647b 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -1424,12 +1424,10 @@ clockcache_get_free_page(clockcache *cc, uint64 end_entry = start_entry + CC_ENTRIES_PER_BATCH; for (entry_no = start_entry; entry_no < end_entry; entry_no++) { entry = &cc->entry[entry_no]; - if (entry->status == CC_FREE_STATUS) { - bool32 reserved = __sync_bool_compare_and_swap( - &entry->status, CC_FREE_STATUS, CC_ALLOC_STATUS); - if (!reserved) { - continue; - } + if (entry->status == CC_FREE_STATUS + && __sync_bool_compare_and_swap( + &entry->status, CC_FREE_STATUS, CC_ALLOC_STATUS)) + { // A page that begins dirty (a fresh allocation) must carry a dirty // generation, just like a clean->dirty transition. The entry is From 72315327112646e52c34692402ce23c6d98f4af5 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sat, 18 Jul 2026 00:15:39 -0700 Subject: [PATCH 11/64] formatting Signed-off-by: Rob Johnson --- src/log.h | 4 +--- src/memtable.c | 7 +++--- src/mini_allocator.c | 40 +++++++++++++++-------------------- src/mini_allocator.h | 12 +++++------ src/rc_allocator.c | 36 ++++++++++++++----------------- src/rc_allocator.h | 4 ++-- src/shard_log.c | 25 +++++++++++----------- src/shard_log.h | 5 ++--- tests/functional/btree_test.c | 28 ++++++++++++------------ tests/functional/cache_test.c | 29 ++++++++++++++----------- tests/unit/splinter_test.c | 10 ++------- 11 files changed, 92 insertions(+), 108 deletions(-) diff --git a/src/log.h b/src/log.h index 3dd27172..30366edc 100644 --- a/src/log.h +++ b/src/log.h @@ -98,9 +98,7 @@ log_seal(log_handle *log) } static inline platform_status -log_rotate(log_handle *log, - log_segment_info *sealed, - log_segment_info *fresh) +log_rotate(log_handle *log, log_segment_info *sealed, log_segment_info *fresh) { return log->ops->rotate(log, sealed, fresh); } diff --git a/src/memtable.c b/src/memtable.c index e8c8d95e..e8b8969b 100644 --- a/src/memtable.c +++ b/src/memtable.c @@ -403,8 +403,7 @@ memtable_context_init_at_generation(memtable_context *ctxt, batch_rwlock_init(&ctxt->rwlock); - for (uint64 generation_offset = 0; - generation_offset < cfg->max_memtables; + for (uint64 generation_offset = 0; generation_offset < cfg->max_memtables; generation_offset++) { uint64 generation = first_generation + generation_offset; @@ -420,8 +419,8 @@ memtable_context_init_at_generation(memtable_context *ctxt, * Otherwise, the checkpoint has incorporated every generation before * first_generation. */ - ctxt->generation_retired = first_generation == 0 ? (uint64)-1 - : first_generation - 1; + ctxt->generation_retired = + first_generation == 0 ? (uint64)-1 : first_generation - 1; ctxt->is_empty = TRUE; diff --git a/src/mini_allocator.c b/src/mini_allocator.c index 3142a6a7..143d374d 100644 --- a/src/mini_allocator.c +++ b/src/mini_allocator.c @@ -925,9 +925,8 @@ mini_prefetch(cache *cc, page_type type, uint64 meta_head) static platform_status mini_recovery_corruption(const char *reason, uint64 addr) { - platform_error_log("Malformed mini allocator metadata: %s (addr=%lu).\n", - reason, - addr); + platform_error_log( + "Malformed mini allocator metadata: %s (addr=%lu).\n", reason, addr); return STATUS_INVALID_STATE; } @@ -936,8 +935,7 @@ mini_recovery_valid_geometry(const allocator_config *cfg) { if (cfg == NULL || cfg->io_cfg == NULL || cfg->capacity == 0 || cfg->io_cfg->page_size == 0 || cfg->io_cfg->extent_size == 0 - || cfg->io_cfg->page_size - < offsetof(mini_meta_hdr, entry_buffer) + || cfg->io_cfg->page_size < offsetof(mini_meta_hdr, entry_buffer) || cfg->io_cfg->extent_size < cfg->io_cfg->page_size || cfg->io_cfg->extent_size % cfg->io_cfg->page_size != 0 || cfg->capacity % cfg->io_cfg->extent_size != 0) @@ -990,8 +988,8 @@ mini_recovery_validate_meta_header(const mini_meta_hdr *hdr, meta_addr); } - uint64 expected_pos = first_entry_offset - + (uint64)hdr->num_entries * sizeof(meta_entry); + uint64 expected_pos = + first_entry_offset + (uint64)hdr->num_entries * sizeof(meta_entry); if (hdr->pos != expected_pos) { return mini_recovery_corruption("metadata entry position is invalid", meta_addr); @@ -1037,10 +1035,10 @@ mini_recovery_walk(cache *cc, uint64 meta_head, page_type meta_type, mini_recovery_visit_fn visit, - void *arg) + void *arg) { - if (cc == NULL || visit == NULL - || meta_type < PAGE_TYPE_FIRST || meta_type >= NUM_PAGE_TYPES) + if (cc == NULL || visit == NULL || meta_type < PAGE_TYPE_FIRST + || meta_type >= NUM_PAGE_TYPES) { return STATUS_BAD_PARAM; } @@ -1096,8 +1094,8 @@ mini_recovery_walk(cache *cc, return STATUS_IO_ERROR; } - mini_meta_hdr *hdr = (mini_meta_hdr *)meta_page->data; - platform_status rc = mini_recovery_validate_meta_header( + mini_meta_hdr *hdr = (mini_meta_hdr *)meta_page->data; + platform_status rc = mini_recovery_validate_meta_header( hdr, cfg->io_cfg->page_size, expected_prev, meta_addr); if (!SUCCESS(rc)) { cache_unget(cc, meta_page); @@ -1105,8 +1103,8 @@ mini_recovery_walk(cache *cc, } uint64 next_meta_addr = hdr->next_meta_addr; - rc = mini_recovery_validate_next_meta_addr( - cfg, meta_addr, next_meta_addr); + rc = + mini_recovery_validate_next_meta_addr(cfg, meta_addr, next_meta_addr); if (!SUCCESS(rc)) { cache_unget(cc, meta_page); return rc; @@ -1116,12 +1114,11 @@ mini_recovery_walk(cache *cc, for (uint64 entry_no = 0; entry_no < hdr->num_entries; entry_no++) { uint64 batch = meta_entry_batch(entry); page_type extent_type = meta_entry_type(entry); - uint64 extent_number = + uint64 extent_number = entry->packed >> (META_ENTRY_BATCH_BITS + META_ENTRY_TYPE_BITS); - if (batch >= MINI_MAX_BATCHES - || extent_type < PAGE_TYPE_FIRST || extent_type >= NUM_PAGE_TYPES - || extent_number == 0 + if (batch >= MINI_MAX_BATCHES || extent_type < PAGE_TYPE_FIRST + || extent_type >= NUM_PAGE_TYPES || extent_number == 0 || extent_number > UINT64_MAX / cfg->io_cfg->extent_size) { cache_unget(cc, meta_page); @@ -1136,11 +1133,8 @@ mini_recovery_walk(cache *cc, extent_addr); } - rc = visit(extent_addr, - extent_type, - MINI_RECOVERY_EXTENT_DATA, - batch, - arg); + rc = visit( + extent_addr, extent_type, MINI_RECOVERY_EXTENT_DATA, batch, arg); if (!SUCCESS(rc)) { cache_unget(cc, meta_page); return rc; diff --git a/src/mini_allocator.h b/src/mini_allocator.h index f596d986..8a655679 100644 --- a/src/mini_allocator.h +++ b/src/mini_allocator.h @@ -148,7 +148,7 @@ typedef enum mini_recovery_extent_kind { MINI_RECOVERY_EXTENT_DATA, } mini_recovery_extent_kind; -#define MINI_RECOVERY_METADATA_BATCH ((uint64)-1) +#define MINI_RECOVERY_METADATA_BATCH ((uint64) - 1) typedef platform_status (*mini_recovery_visit_fn)( uint64 extent_addr, @@ -158,11 +158,11 @@ typedef platform_status (*mini_recovery_visit_fn)( void *arg); platform_status -mini_recovery_walk(cache *cc, - uint64 meta_head, - page_type meta_type, - mini_recovery_visit_fn visit, - void *arg); +mini_recovery_walk(cache *cc, + uint64 meta_head, + page_type meta_type, + mini_recovery_visit_fn visit, + void *arg); /* * mini_meta_cursor: a non-blocking cursor over the extent entries of a diff --git a/src/rc_allocator.c b/src/rc_allocator.c index 7a365fa2..f11a4524 100644 --- a/src/rc_allocator.c +++ b/src/rc_allocator.c @@ -353,7 +353,7 @@ rc_allocator_read_clean_state(rc_allocator *al, } void *page = platform_buffer_getaddr(&buffer); - rc = io_read(al->io, + rc = io_read(al->io, page, al->cfg->io_cfg->page_size, rc_allocator_clean_state_addr(al->cfg, slot)); @@ -437,7 +437,7 @@ static platform_status rc_allocator_publish_clean_state(rc_allocator *al, bool32 clean_shutdown) { rc_allocator_clean_states states; - platform_status rc = rc_allocator_load_clean_states(al, &states); + platform_status rc = rc_allocator_load_clean_states(al, &states); if (!SUCCESS(rc)) { return rc; } @@ -453,9 +453,8 @@ rc_allocator_publish_clean_state(rc_allocator *al, bool32 clean_shutdown) ZERO_CONTENTS(&state); state.magic = RC_ALLOCATOR_CLEAN_STATE_MAGIC; state.format_version = RC_ALLOCATOR_CLEAN_STATE_VERSION; - state.sequence = states.have_newest - ? states.state[states.newest_slot].sequence + 1 - : 1; + state.sequence = + states.have_newest ? states.state[states.newest_slot].sequence + 1 : 1; state.clean_shutdown = clean_shutdown; state.checksum = rc_allocator_clean_state_checksum(&state); return rc_allocator_write_clean_state(al, target_slot, &state, TRUE); @@ -507,17 +506,14 @@ rc_allocator_recovery_initialize_refcounts(rc_allocator *al) return STATUS_BAD_PARAM; } - memset(al->ref_count, - 0, - rc_allocator_refcount_buffer_size(al->cfg)); + memset(al->ref_count, 0, rc_allocator_refcount_buffer_size(al->cfg)); /* * Extent 0 contains both the allocator meta page and every fixed table * superblock. The refcount table begins at extent 1; the two extents * after it hold alternating clean-state records. */ - for (uint64 extent_no = 0; extent_no < reserved_extent_count; extent_no++) - { + for (uint64 extent_no = 0; extent_no < reserved_extent_count; extent_no++) { platform_assert(al->ref_count[extent_no] == AL_FREE); al->ref_count[extent_no] = AL_ONE_REF; rc_allocator_record_allocated_extent(al); @@ -530,7 +526,7 @@ rc_allocator_recovery_initialize_refcounts(rc_allocator *al) static bool32 rc_allocator_recovery_extent_is_reserved(const rc_allocator *al, - uint64 extent_no) + uint64 extent_no) { return extent_no < rc_allocator_reserved_extent_count(al->cfg); } @@ -603,8 +599,8 @@ rc_allocator_init_meta_page(rc_allocator *al) memset(al->meta_page->splinters, INVALID_ALLOCATOR_ROOT_ID, sizeof(al->meta_page->splinters)); - al->meta_page->geometry = rc_allocator_config_get_disk_geometry(al->cfg); - al->meta_page->format_magic = RC_ALLOCATOR_FORMAT_MAGIC; + al->meta_page->geometry = rc_allocator_config_get_disk_geometry(al->cfg); + al->meta_page->format_magic = RC_ALLOCATOR_FORMAT_MAGIC; al->meta_page->format_version = RC_ALLOCATOR_FORMAT_VERSION; return STATUS_OK; @@ -894,7 +890,8 @@ rc_allocator_mount_internal(rc_allocator *al, al->recovery_in_progress = TRUE; } else { // Load the ref counts from disk during a normal, clean mount. - status = io_read(io, al->ref_count, buffer_size, cfg->io_cfg->extent_size); + status = + io_read(io, al->ref_count, buffer_size, cfg->io_cfg->extent_size); if (!SUCCESS(status)) { goto deinit_buffer; } @@ -968,8 +965,8 @@ rc_allocator_rebuild_acquire_extent(rc_allocator *al, uint64 extent_addr) } while (TRUE) { - refcount old_ref = __atomic_load_n(&al->ref_count[extent_no], - __ATOMIC_RELAXED); + refcount old_ref = + __atomic_load_n(&al->ref_count[extent_no], __ATOMIC_RELAXED); if (old_ref == (refcount)-1) { platform_error_log("Allocator recovery refcount overflow for extent " "%lu.\n", @@ -977,8 +974,7 @@ rc_allocator_rebuild_acquire_extent(rc_allocator *al, uint64 extent_addr) return STATUS_LIMIT_EXCEEDED; } - refcount new_ref = - old_ref == AL_FREE ? AL_ONE_REF : old_ref + 1; + refcount new_ref = old_ref == AL_FREE ? AL_ONE_REF : old_ref + 1; if (!__sync_bool_compare_and_swap( &al->ref_count[extent_no], old_ref, new_ref)) { @@ -1172,7 +1168,7 @@ rc_allocator_alloc_super_addr(rc_allocator *al, // assign the first available slot and update the on disk metadata. al->meta_page->splinters[idx] = allocator_root_id; *addr = (1 + idx) * al->cfg->io_cfg->page_size; - platform_status io_status = rc_allocator_write_meta_page(al); + platform_status io_status = rc_allocator_write_meta_page(al); platform_assert_status_ok(io_status); status = STATUS_OK; break; @@ -1196,7 +1192,7 @@ rc_allocator_remove_super_addr(rc_allocator *al, */ if (al->meta_page->splinters[idx] == allocator_root_id) { al->meta_page->splinters[idx] = INVALID_ALLOCATOR_ROOT_ID; - platform_status status = rc_allocator_write_meta_page(al); + platform_status status = rc_allocator_write_meta_page(al); platform_assert_status_ok(status); platform_mutex_unlock(&al->lock); return; diff --git a/src/rc_allocator.h b/src/rc_allocator.h index 079e565a..28f8226c 100644 --- a/src/rc_allocator.h +++ b/src/rc_allocator.h @@ -39,8 +39,8 @@ typedef struct ONDISK rc_allocator_meta_page { * that the two fixed clean-state extents after the refcount map are owned * by this allocator rather than by an older on-disk format. */ - uint64 format_magic; - uint64 format_version; + uint64 format_magic; + uint64 format_version; allocator_root_id splinters[RC_ALLOCATOR_MAX_ROOT_IDS]; checksum128 checksum; } rc_allocator_meta_page; diff --git a/src/shard_log.c b/src/shard_log.c index f9096323..1db98edd 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -25,15 +25,16 @@ static uint64 shard_log_magic_idx = 0; -int shard_log_write(log_handle *log, - key tuple_key, - message msg, - uint64 memtable_generation, - uint64 leaf_generation); +int +shard_log_write(log_handle *log, + key tuple_key, + message msg, + uint64 memtable_generation, + uint64 leaf_generation); platform_status shard_log_seal(log_handle *log); platform_status -shard_log_rotate(log_handle *log, +shard_log_rotate(log_handle *log, log_segment_info *sealed, log_segment_info *fresh); void @@ -182,7 +183,7 @@ struct ONDISK log_entry { ondisk_tuple tuple; }; -#define INVALID_LOG_GENERATION ((uint64)-1) +#define INVALID_LOG_GENERATION ((uint64) - 1) static key log_entry_key(log_entry *le) @@ -254,7 +255,7 @@ get_new_page_for_thread(shard_log *log, { uint64 next_extent; - *page = shard_log_alloc(log, &next_extent); + *page = shard_log_alloc(log, &next_extent); if (*page == NULL) { return -1; } @@ -426,9 +427,8 @@ shard_log_seal(log_handle *logh) debug_assert(thread_data->offset >= sizeof(shard_log_hdr)); debug_assert(thread_data->offset <= shard_log_page_size(log->cfg)); - shard_log_hdr *hdr = (shard_log_hdr *)page->data; - log_entry *cursor = - (log_entry *)(page->data + thread_data->offset); + shard_log_hdr *hdr = (shard_log_hdr *)page->data; + log_entry *cursor = (log_entry *)(page->data + thread_data->offset); uint64 free_space = shard_log_page_size(log->cfg) - thread_data->offset; if (sizeof(log_entry) <= free_space) { log_entry_set_terminal(cursor); @@ -510,8 +510,7 @@ shard_log_rotate(log_handle *logh, } void -shard_log_segment_discard(cache *cc, - const log_segment_info *segment) +shard_log_segment_discard(cache *cc, const log_segment_info *segment) { if (segment->meta_addr == 0) { return; diff --git a/src/shard_log.h b/src/shard_log.h index 3e024d14..086381a7 100644 --- a/src/shard_log.h +++ b/src/shard_log.h @@ -47,7 +47,7 @@ typedef struct shard_log { uint64 meta_head; uint64 magic; /* Set once any log page has been allocated; survives sealing. */ - bool32 has_pages; + bool32 has_pages; } shard_log; typedef struct log_entry log_entry; @@ -92,8 +92,7 @@ shard_log_zap(shard_log *log); * descriptors will eventually own and release this reference instead. */ void -shard_log_segment_discard(cache *cc, - const log_segment_info *segment); +shard_log_segment_discard(cache *cc, const log_segment_info *segment); platform_status shard_log_iterator_init(cache *cc, diff --git a/tests/functional/btree_test.c b/tests/functional/btree_test.c index 139db99e..6d4e6d7a 100644 --- a/tests/functional/btree_test.c +++ b/tests/functional/btree_test.c @@ -76,9 +76,9 @@ test_memtable_generation_init(cache *cc, test_btree_config *cfg, platform_heap_id hid) { - const uint64 first_generation = 17; - const uint64 max_memtables = 4; - memtable_config mt_cfg = *cfg->mt_cfg; + const uint64 first_generation = 17; + const uint64 max_memtables = 4; + memtable_config mt_cfg = *cfg->mt_cfg; memtable_context mt_ctxt; mt_cfg.max_memtables = max_memtables; @@ -116,12 +116,12 @@ test_memtable_generation_init(cache *cc, } rc = memtable_context_init_at_generation(&mt_ctxt, - hid, - cc, - &mt_cfg, - test_btree_process_noop, - NULL, - first_generation); + hid, + cc, + &mt_cfg, + test_btree_process_noop, + NULL, + first_generation); if (!SUCCESS(rc)) { return rc; } @@ -156,7 +156,8 @@ test_memtable_generation_init(cache *cc, deinit_recovery: memtable_context_deinit(&mt_ctxt); if (SUCCESS(rc)) { - platform_default_log("btree_test: memtable generation init test passed\n"); + platform_default_log( + "btree_test: memtable generation init test passed\n"); } return rc; } @@ -206,10 +207,9 @@ test_mini_recovery_walk(cache *cc) allocator *al = cache_get_allocator(cc); mini_allocator mini; test_mini_recovery_walk_state state; - uint64 meta_head = 0; + uint64 meta_head = 0; uint64 data_extent = 0; - platform_status rc = - allocator_alloc(al, &meta_head, PAGE_TYPE_MISC); + platform_status rc = allocator_alloc(al, &meta_head, PAGE_TYPE_MISC); if (!SUCCESS(rc)) { return rc; } @@ -238,7 +238,7 @@ test_mini_recovery_walk(cache *cc) ZERO_CONTENTS(&state); state.fail_data_visit = TRUE; - rc = mini_recovery_walk( + rc = mini_recovery_walk( cc, meta_head, PAGE_TYPE_MISC, test_mini_recovery_walk_visit, &state); if (!STATUS_IS_EQ(rc, STATUS_TEST_FAILED) || state.metadata_visits != 1 || state.data_visits != 1) diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index 7e4a06b0..8941fe2a 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -784,13 +784,12 @@ test_cache_writeback_fence_liveness(cache *cc, clockcache_config *cfg, platform_heap_id hid) { - platform_status rc = STATUS_OK; - uint64 page_size = cache_config_page_size(&cfg->super); - uint64 pages_per_extent = - cache_config_pages_per_extent(&cfg->super); - uint64 *addr_arr = NULL; - const uint8 baseline = 0x11, target_value = 0x77; - bool32 hammer_started = FALSE, fence_started = FALSE; + platform_status rc = STATUS_OK; + uint64 page_size = cache_config_page_size(&cfg->super); + uint64 pages_per_extent = cache_config_pages_per_extent(&cfg->super); + uint64 *addr_arr = NULL; + const uint8 baseline = 0x11, target_value = 0x77; + bool32 hammer_started = FALSE, fence_started = FALSE; cache_hammer_context hammer_ctxt = {.cc = cc, .stop = FALSE}; cache_fence_test_context fence_ctxt = { .cc = cc, .finished = FALSE, .status = STATUS_OK}; @@ -833,8 +832,11 @@ test_cache_writeback_fence_liveness(cache *cc, } hammer_started = TRUE; - rc = platform_thread_create( - &fence_thread, FALSE, cache_test_writeback_fence_thread, &fence_ctxt, hid); + rc = platform_thread_create(&fence_thread, + FALSE, + cache_test_writeback_fence_thread, + &fence_ctxt, + hid); if (!SUCCESS(rc)) { goto cleanup; } @@ -876,7 +878,8 @@ test_cache_writeback_fence_liveness(cache *cc, rc = cache_durable_barrier(cc); } if (SUCCESS(rc)) { - rc = cache_test_verify_disk_page(cc, addr_arr[0], page_size, target_value); + rc = + cache_test_verify_disk_page(cc, addr_arr[0], page_size, target_value); } if (addr_arr != NULL) { @@ -894,9 +897,11 @@ test_cache_writeback_fence_liveness(cache *cc, } if (SUCCESS(rc)) { - platform_default_log("cache_test: writeback fence liveness test passed\n"); + platform_default_log( + "cache_test: writeback fence liveness test passed\n"); } else { - platform_default_log("cache_test: writeback fence liveness test failed\n"); + platform_default_log( + "cache_test: writeback fence liveness test failed\n"); } return rc; } diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index 9c550ec0..01fe15f6 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -259,7 +259,7 @@ CTEST2(splinter, test_inserts) */ CTEST2(splinter, test_mount_rejects_newer_active_checkpoint) { - allocator *alp = (allocator *)&data->al; + allocator *alp = (allocator *)&data->al; allocator_root_id root_id = test_generate_allocator_root_id(); core_handle created, mounted, rejected, cleanup; platform_status rc; @@ -277,13 +277,7 @@ CTEST2(splinter, test_mount_rejects_newer_active_checkpoint) DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); merge_accumulator msg; merge_accumulator_init(&msg, data->hid); - test_key(&keybuf, - TEST_RANDOM, - 1, - 0, - 0, - data->workload_cfg->key_size, - 0); + test_key(&keybuf, TEST_RANDOM, 1, 0, 0, data->workload_cfg->key_size, 0); generate_test_message(&data->gen, 1, &msg); rc = core_insert(&created, key_buffer_key(&keybuf), From 96b0b60e39cf6e8a8c23b78f621377f03064a471 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sat, 18 Jul 2026 19:15:20 -0700 Subject: [PATCH 12/64] Cleaning up trunk Signed-off-by: Rob Johnson --- src/core.c | 56 +++++++++++++++++--------- src/trunk.c | 114 +++++++++++++++++++--------------------------------- src/trunk.h | 49 ++++++++++------------ 3 files changed, 98 insertions(+), 121 deletions(-) diff --git a/src/core.c b/src/core.c index 9af1975a..d17af7bc 100644 --- a/src/core.c +++ b/src/core.c @@ -467,12 +467,8 @@ core_destroy_checkpoint_storage(core_handle *spl) if (!records.valid[slot] || records.record[slot].root_addr == 0) { continue; } - rc = trunk_dec_ref(spl->cfg.trunk_node_cfg, - PROCESS_PRIVATE_HEAP_ID, - spl->cc, - spl->al, - spl->ts, - records.record[slot].root_addr); + trunk_snapshot old_snapshot = {.root_addr = records.record[slot].root_addr}; + rc = trunk_snapshot_release(&spl->trunk_context, &old_snapshot); if (!SUCCESS(rc)) { platform_error_log("core_destroy_checkpoint_storage: failed to " "release record %lu root %lu: %s\n", @@ -506,7 +502,7 @@ core_capture_checkpoint_cut(core_handle *spl, */ memtable_block_lookups(&spl->mt_ctxt); uint64 retired_generation = memtable_generation_retired(&spl->mt_ctxt); - platform_status rc = trunk_snapshot_acquire(&spl->trunk_context, snapshot); + platform_status rc = trunk_snapshot_create(&spl->trunk_context, snapshot); memtable_unblock_lookups(&spl->mt_ctxt); if (!SUCCESS(rc)) { return rc; @@ -558,7 +554,11 @@ core_publish_checkpoint_record(core_handle *spl, * persist newer log/data pages, but it does not seal or publish a logical * durable-log tail; tail sync is a separate operation. */ - rc = trunk_make_durable(&spl->trunk_context); + rc = cache_writeback_dirty(spl->cc); + if (!SUCCESS(rc)) { + goto release_snapshot; + } + rc = cache_durable_barrier(spl->cc); if (!SUCCESS(rc)) { goto release_snapshot; } @@ -628,15 +628,12 @@ core_publish_checkpoint_record(core_handle *spl, } if (old_root_addr != 0) { - rc = trunk_dec_ref(spl->cfg.trunk_node_cfg, - PROCESS_PRIVATE_HEAP_ID, - spl->cc, - spl->al, - spl->ts, - old_root_addr); + trunk_snapshot old_snapshot = {.root_addr = old_root_addr}; + rc = trunk_snapshot_release(&spl->trunk_context, &old_snapshot); if (!SUCCESS(rc)) { - platform_error_log("core_publish_checkpoint_record: trunk_dec_ref " - "failed for old root addr %lu: %s\n", + platform_error_log("core_publish_checkpoint_record: " + "trunk_snapshot_release failed for old root addr " + "%lu: %s\n", old_root_addr, platform_status_to_string(rc)); goto unlock_checkpoint; @@ -2335,8 +2332,13 @@ core_mkfs(core_handle *spl, } } - rc = trunk_context_init( - &spl->trunk_context, spl->cfg.trunk_node_cfg, hid, cc, al, ts, 0); + rc = trunk_context_init(&spl->trunk_context, + spl->cfg.trunk_node_cfg, + hid, + cc, + al, + ts, + (trunk_snapshot){.root_addr = 0}); if (!SUCCESS(rc)) { platform_error_log("core_mkfs: trunk_context_init failed: %s\n", platform_status_to_string(rc)); @@ -2473,8 +2475,22 @@ core_mount(core_handle *spl, } } - rc = trunk_context_init( - &spl->trunk_context, spl->cfg.trunk_node_cfg, hid, cc, al, ts, root_addr); + trunk_snapshot root_snapshot; + rc = trunk_snapshot_create_from_addr(al, root_addr, &root_snapshot); + if (!SUCCESS(rc)) { + platform_error_log( + "core_mount: trunk_snapshot_create_from_addr failed: %s\n", + platform_status_to_string(rc)); + goto deinit_log; + } + + rc = trunk_context_init(&spl->trunk_context, + spl->cfg.trunk_node_cfg, + hid, + cc, + al, + ts, + root_snapshot); if (!SUCCESS(rc)) { platform_error_log("core_mount: trunk_context_init failed: %s\n", platform_status_to_string(rc)); diff --git a/src/trunk.c b/src/trunk.c index 8ebbc0b2..4ea8bdb9 100644 --- a/src/trunk.c +++ b/src/trunk.c @@ -2546,20 +2546,32 @@ trunk_read_end(trunk_context *context) * to take a snapshot. */ platform_status -trunk_snapshot_acquire(trunk_context *context, trunk_snapshot *snapshot) +trunk_snapshot_create(trunk_context *context, trunk_snapshot *snapshot) { snapshot->root_addr = 0; trunk_read_begin(context); if (context->root != NULL) { snapshot->root_addr = context->root->ref.addr; - trunk_inc_ref(context->al, snapshot->root_addr); + trunk_addr_inc_ref(context->al, snapshot->root_addr); } trunk_read_end(context); return STATUS_OK; } +platform_status +trunk_snapshot_create_from_addr(allocator *al, + uint64 root_addr, + trunk_snapshot *snapshot) +{ + snapshot->root_addr = root_addr; + if (root_addr != 0) { + trunk_addr_inc_ref(al, root_addr); + } + return STATUS_OK; +} + platform_status trunk_snapshot_release(trunk_context *context, trunk_snapshot *snapshot) { @@ -2567,16 +2579,23 @@ trunk_snapshot_release(trunk_context *context, trunk_snapshot *snapshot) return STATUS_OK; } - platform_status rc = trunk_dec_ref(context->cfg, - context->hid, - context->cc, - context->al, - context->ts, - snapshot->root_addr); - if (SUCCESS(rc)) { - snapshot->root_addr = 0; + /* + * Route the release through a scratch context so the ordinary COW + * teardown path (trunk_context_deinit -> trunk_ondisk_node_ref_destroy) + * performs the single decrement snapshot's reference is owed. The scratch + * context only needs context's shared cfg/cc/al/ts; it does not need to + * share context's root. + */ + trunk_context scratch; + platform_status rc = trunk_context_init( + &scratch, context->cfg, context->hid, context->cc, context->al, + context->ts, *snapshot); + snapshot->root_addr = 0; // trunk_context_init consumes it regardless of rc + if (!SUCCESS(rc)) { + return rc; } - return rc; + trunk_context_deinit(&scratch); + return STATUS_OK; } platform_status @@ -6743,7 +6762,7 @@ trunk_context_init(trunk_context *context, cache *cc, allocator *al, task_system *ts, - uint64 root_addr) + trunk_snapshot snapshot) { memset(context, 0, sizeof(trunk_context)); @@ -6753,12 +6772,20 @@ trunk_context_init(trunk_context *context, context->al = al; context->ts = ts; - if (root_addr != 0) { + if (snapshot.root_addr != 0) { + // Adopt: snapshot already carries an owned reference (from + // trunk_snapshot_create or trunk_snapshot_create_from_addr), so this + // does not take a new one; it just gives that reference a home. context->root = trunk_ondisk_node_ref_create( - context, NEGATIVE_INFINITY_KEY, root_addr, FALSE); + context, NEGATIVE_INFINITY_KEY, snapshot.root_addr, TRUE /* adopt */); if (context->root == NULL) { platform_error_log("trunk_node_context_init: " "ondisk_node_ref_create failed\n"); + // The reference was never actually adopted (creation failed before + // that could happen), but the caller has already relinquished it. + // Discharge it here so this function always consumes its snapshot, + // regardless of outcome. + trunk_ondisk_node_dec_ref(context, snapshot.root_addr); return STATUS_NO_MEMORY; } } @@ -6784,34 +6811,6 @@ trunk_context_init(trunk_context *context, return STATUS_OK; } -void -trunk_inc_ref(allocator *al, uint64 root_addr) -{ - trunk_addr_inc_ref(al, root_addr); -} - -platform_status -trunk_dec_ref(const trunk_config *cfg, - platform_heap_id hid, - cache *cc, - allocator *al, - task_system *ts, - uint64 root_addr) -{ - trunk_context context; - platform_status rc = - trunk_context_init(&context, cfg, hid, cc, al, ts, root_addr); - if (!SUCCESS(rc)) { - platform_error_log("trunk_node_dec_ref: trunk_node_context_init failed: " - "%d\n", - rc.r); - return rc; - } - trunk_ondisk_node_dec_ref(&context, root_addr); - trunk_context_deinit(&context); - return STATUS_OK; -} - void trunk_context_deinit(trunk_context *context) { @@ -6828,37 +6827,6 @@ trunk_context_deinit(trunk_context *context) } } - -platform_status -trunk_context_clone(trunk_context *dst, trunk_context *src) -{ - platform_status rc; - trunk_ondisk_node_handle handle; - rc = trunk_init_root_handle(src, &handle); - if (!SUCCESS(rc)) { - platform_error_log("trunk_node_context_clone: trunk_init_root_handle " - "failed: %d\n", - rc.r); - return rc; - } - uint64 root_addr = handle.header_page->disk_addr; - - rc = trunk_context_init( - dst, src->cfg, src->hid, src->cc, src->al, src->ts, root_addr); - trunk_ondisk_node_handle_deinit(&handle); - return rc; -} - -platform_status -trunk_make_durable(trunk_context *context) -{ - platform_status rc = cache_writeback_dirty(context->cc); - if (!SUCCESS(rc)) { - return rc; - } - return cache_durable_barrier(context->cc); -} - /************************************ * Stats ************************************/ diff --git a/src/trunk.h b/src/trunk.h index a48dbab0..70786bec 100644 --- a/src/trunk.h +++ b/src/trunk.h @@ -205,6 +205,14 @@ trunk_config_init(trunk_config *config, uint64 prefetch_budget, bool32 use_stats); +/* + * Initializes context with a root of snapshot. Consumes snapshot's owned + * reference regardless of outcome: on success the reference now lives in + * context (released by a later trunk_context_deinit); on failure it has + * already been discharged. Either way, the caller must not use or release + * snapshot again. A null snapshot (root_addr == 0) starts an empty trunk with + * no root, as for a freshly formatted table. + */ platform_status trunk_context_init(trunk_context *context, const trunk_config *cfg, @@ -212,45 +220,35 @@ trunk_context_init(trunk_context *context, cache *cc, allocator *al, task_system *ts, - uint64 root_addr); - -void -trunk_inc_ref(allocator *al, uint64 root_addr); - -platform_status -trunk_dec_ref(const trunk_config *cfg, - platform_heap_id hid, - cache *cc, - allocator *al, - task_system *ts, - uint64 root_addr); + trunk_snapshot snapshot); void trunk_context_deinit(trunk_context *context); -/* Create a writable snapshot of a trunk */ +/* Capture an owned reference to the current COW root without reading it. */ platform_status -trunk_context_clone(trunk_context *dst, trunk_context *src); +trunk_snapshot_create(trunk_context *context, trunk_snapshot *snapshot); -/* Capture an owned reference to the current COW root without reading it. */ +/* + * Capture an owned reference to a root_addr that is not (or not yet) any live + * context's root -- e.g. one just read out of a durable checkpoint record. + * root_addr must already be referenced by the caller (its persisted refcount + * accounts for it); this takes out an additional, independent reference. A + * null root_addr (0) produces a null snapshot without touching the allocator. + */ platform_status -trunk_snapshot_acquire(trunk_context *context, trunk_snapshot *snapshot); +trunk_snapshot_create_from_addr(allocator *al, + uint64 root_addr, + trunk_snapshot *snapshot); /* Drop an owned snapshot reference that was not published. */ platform_status trunk_snapshot_release(trunk_context *context, trunk_snapshot *snapshot); -/* Make a trunk durable */ -platform_status -trunk_make_durable(trunk_context *context); - /******************************** * Mutations ********************************/ -void -trunk_modification_begin(trunk_context *context); - // Build a new trunk with the branch incorporated. The new trunk is not yet // visible to queriers. platform_status @@ -276,8 +274,6 @@ trunk_optimize(trunk_context *context, bool32 full_leaf_compactions, struct splinterdb_notification *notification); -void -trunk_modification_end(trunk_context *context); /******************************** * Queries @@ -287,9 +283,6 @@ platform_status trunk_init_root_handle(trunk_context *context, trunk_ondisk_node_handle *handle); -uint64 -trunk_ondisk_node_handle_addr(const trunk_ondisk_node_handle *handle); - void trunk_ondisk_node_handle_deinit(trunk_ondisk_node_handle *handle); From 7b8b84431f0294859d94e772be2fb2cb6ea3df6a Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sun, 19 Jul 2026 00:08:47 -0700 Subject: [PATCH 13/64] clean up mini allocator recovery Signed-off-by: Rob Johnson --- src/allocator.h | 23 ++- src/mini_allocator.c | 310 +++++++++++++++++++++------------- src/mini_allocator.h | 56 ++---- src/rc_allocator.c | 21 ++- src/rc_allocator.h | 9 +- tests/functional/btree_test.c | 104 ------------ tests/functional/cache_test.c | 188 ++++++++++++++++++++- 7 files changed, 439 insertions(+), 272 deletions(-) diff --git a/src/allocator.h b/src/allocator.h index eb83b631..54c8b939 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -136,6 +136,16 @@ typedef platform_status (*alloc_fn)(allocator *al, typedef refcount (*dec_ref_fn)(allocator *al, uint64 addr, page_type type); typedef refcount (*generic_ref_fn)(allocator *al, uint64 addr); +/* + * Record one logical reference to addr while rebuilding a recovery map (see + * rc_allocator_mount_recovery()). The first reference to a given extent + * establishes its nonzero allocation floor; later references increment it + * normally. addr must be the base address of a non-reserved extent. + */ +typedef platform_status (*recovery_record_reference_fn)(allocator *al, + uint64 addr, + page_type type); + typedef platform_status (*get_super_addr_fn)(allocator *al, allocator_root_id spl_id, uint64 *addr); @@ -157,9 +167,10 @@ typedef struct allocator_ops { allocator_get_config_fn get_config; alloc_fn alloc; - generic_ref_fn inc_ref; - dec_ref_fn dec_ref; - generic_ref_fn get_ref; + generic_ref_fn inc_ref; + dec_ref_fn dec_ref; + generic_ref_fn get_ref; + recovery_record_reference_fn recovery_record_reference; alloc_super_addr_fn alloc_super_addr; get_super_addr_fn get_super_addr; @@ -211,6 +222,12 @@ allocator_get_refcount(allocator *al, uint64 addr) return al->ops->get_ref(al, addr); } +static inline platform_status +allocator_recovery_record_reference(allocator *al, uint64 addr, page_type type) +{ + return al->ops->recovery_record_reference(al, addr, type); +} + static inline platform_status allocator_get_super_addr(allocator *al, allocator_root_id spl_id, uint64 *addr) { diff --git a/src/mini_allocator.c b/src/mini_allocator.c index 143d374d..6832604e 100644 --- a/src/mini_allocator.c +++ b/src/mini_allocator.c @@ -760,35 +760,78 @@ mini_deinit(cache *cc, uint64 meta_head, page_type type) *----------------------------------------------------------------------------- * mini_for_each_meta_page -- * - * Calls func on each meta_page in the mini_allocator. + * Walks the meta_page chain of the mini_allocator, calling func on each + * meta_page. + * + * If before is non-NULL, it is called once per distinct meta extent -- + * before that extent's first page is cache_get()'d -- with just the + * extent's address. This lets a caller that does not yet have allocator + * ownership of these pages (e.g. crash recovery, rebuilding the refcount + * table from scratch) establish it first, before cache_get()'s debug + * allocation checks run. Ordinary callers, which already own everything + * they are walking, pass NULL. + * + * func and before may fail; the walk stops and returns the first failure. * * Results: - * None + * STATUS_OK, the first failure from before/func, or STATUS_IO_ERROR if a + * meta page could not be read. * * Side effects: - * func may store output in arg. + * func/before may store output in arg. *----------------------------------------------------------------------------- */ -typedef void (*mini_for_each_meta_page_fn)(cache *cc, - page_type type, - page_handle *meta_page, - void *arg); +typedef platform_status (*mini_meta_extent_fn)(cache *cc, + page_type type, + uint64 extent_addr, + void *arg); -static void +typedef platform_status (*mini_for_each_meta_page_fn)(cache *cc, + page_type type, + page_handle *meta_page, + void *arg); + +static platform_status mini_for_each_meta_page(cache *cc, uint64 meta_head, page_type type, + mini_meta_extent_fn before, mini_for_each_meta_page_fn func, void *arg) { - uint64 meta_addr = meta_head; + uint64 meta_addr = meta_head; + uint64 extent_size = cache_extent_size(cc); + uint64 prior_extent = (uint64)-1; + while (meta_addr != 0) { + if (before != NULL) { + uint64 extent_addr = meta_addr - meta_addr % extent_size; + if (extent_addr != prior_extent) { + platform_status rc = before(cc, type, extent_addr, arg); + if (!SUCCESS(rc)) { + return rc; + } + prior_extent = extent_addr; + } + } + page_handle *meta_page = cache_get(cc, meta_addr, TRUE, type); - func(cc, type, meta_page, arg); - meta_addr = mini_get_next_meta_addr(meta_page); + if (meta_page == NULL) { + return STATUS_IO_ERROR; + } + + platform_status rc = + func == NULL ? STATUS_OK : func(cc, type, meta_page, arg); + uint64 next_meta_addr = + SUCCESS(rc) ? mini_get_next_meta_addr(meta_page) : 0; cache_unget(cc, meta_page); + if (!SUCCESS(rc)) { + return rc; + } + meta_addr = next_meta_addr; } + return STATUS_OK; } /* mini_for_each(): call a function on each allocated extent in the @@ -804,7 +847,7 @@ typedef struct for_each_func { void *arg; } for_each_func; -static void +static platform_status mini_for_each_meta_page_func(cache *cc, page_type type, page_handle *meta_page, @@ -821,6 +864,7 @@ mini_for_each_meta_page_func(cache *cc, fef->arg); entry = next_entry(entry); } + return STATUS_OK; } static void @@ -832,7 +876,7 @@ mini_for_each(cache *cc, { for_each_func fef = {func, out}; mini_for_each_meta_page( - cc, meta_head, type, mini_for_each_meta_page_func, &fef); + cc, meta_head, type, NULL, mini_for_each_meta_page_func, &fef); } @@ -918,7 +962,7 @@ mini_prefetch(cache *cc, page_type type, uint64 meta_head) /* * ----------------------------------------------------------------------------- - * mini_recovery_walk -- Read-only mini-allocator extent enumeration. + * mini_recover_allocations -- crash-recovery allocator-reference rebuild. * ----------------------------------------------------------------------------- */ @@ -964,12 +1008,6 @@ mini_recovery_valid_extent_addr(const allocator_config *cfg, uint64 addr) && addr <= cfg->capacity - extent_size; } -static uint64 -mini_recovery_extent_base_addr(const allocator_config *cfg, uint64 addr) -{ - return addr - addr % cfg->io_cfg->extent_size; -} - static platform_status mini_recovery_validate_meta_header(const mini_meta_hdr *hdr, uint64 page_size, @@ -1030,125 +1068,160 @@ mini_recovery_validate_next_meta_addr(const allocator_config *cfg, return STATUS_OK; } -platform_status -mini_recovery_walk(cache *cc, - uint64 meta_head, - page_type meta_type, - mini_recovery_visit_fn visit, - void *arg) +typedef struct mini_recover_state { + allocator *al; + allocator_config *cfg; + uint64 expected_prev; + uint64 page_count; + uint64 max_meta_pages; +} mini_recover_state; + +/* + * The "before" hook mini_for_each_meta_page() calls once per distinct meta + * extent, before that extent's first page is cache_get()'d. Recording the + * reference here -- ahead of the read -- is what lets a rebuilding allocator + * (which does not yet consider this extent allocated) satisfy cache_get()'s + * debug allocation check. + */ +static platform_status +mini_recover_visit_meta_extent(cache *cc, + page_type type, + uint64 extent_addr, + void *arg) { - if (cc == NULL || visit == NULL || meta_type < PAGE_TYPE_FIRST - || meta_type >= NUM_PAGE_TYPES) - { - return STATUS_BAD_PARAM; + mini_recover_state *state = (mini_recover_state *)arg; + + if (!mini_recovery_valid_extent_addr(state->cfg, extent_addr)) { + return mini_recovery_corruption("metadata extent is out of range", + extent_addr); } + return allocator_recovery_record_reference(state->al, extent_addr, type); +} - allocator *al = cache_get_allocator(cc); - if (al == NULL) { - return STATUS_BAD_PARAM; +/* + * The "func" hook mini_for_each_meta_page() calls once per meta page, with the + * page held. Validates the page and every data-extent entry on it before + * trusting any of it, and records a reference for each data extent found (its + * type is self-describing on disk, unlike a meta page's). + */ +static platform_status +mini_recover_visit_meta_page(cache *cc, + page_type type, + page_handle *meta_page, + void *arg) +{ + mini_recover_state *state = (mini_recover_state *)arg; + uint64 meta_addr = meta_page->disk_addr; + + if (state->page_count == state->max_meta_pages) { + return mini_recovery_corruption("metadata chain exceeds disk pages", + meta_addr); } - allocator_config *cfg = allocator_get_config(al); - if (!mini_recovery_valid_geometry(cfg)) { - return STATUS_BAD_PARAM; + state->page_count++; + + mini_meta_hdr *hdr = (mini_meta_hdr *)meta_page->data; + platform_status rc = mini_recovery_validate_meta_header( + hdr, state->cfg->io_cfg->page_size, state->expected_prev, meta_addr); + if (!SUCCESS(rc)) { + return rc; } - if (!mini_recovery_valid_page_addr(cfg, meta_head)) { - return mini_recovery_corruption("metadata head is not a page", meta_head); + + rc = mini_recovery_validate_next_meta_addr( + state->cfg, meta_addr, hdr->next_meta_addr); + if (!SUCCESS(rc)) { + return rc; } - uint64 meta_addr = meta_head; - uint64 expected_prev = 0; - uint64 prior_meta_extent = (uint64)-1; - uint64 max_meta_pages = cfg->capacity / cfg->io_cfg->page_size; + meta_entry *entry = first_entry(meta_page); + for (uint64 entry_no = 0; entry_no < hdr->num_entries; entry_no++) { + uint64 batch = meta_entry_batch(entry); + page_type extent_type = meta_entry_type(entry); + uint64 extent_number = + entry->packed >> (META_ENTRY_BATCH_BITS + META_ENTRY_TYPE_BITS); - for (uint64 page_count = 0; meta_addr != 0; page_count++) { - if (page_count == max_meta_pages) { - return mini_recovery_corruption("metadata chain exceeds disk pages", + if (batch >= MINI_MAX_BATCHES || extent_type < PAGE_TYPE_FIRST + || extent_type >= NUM_PAGE_TYPES || extent_number == 0 + || extent_number > UINT64_MAX / state->cfg->io_cfg->extent_size) + { + return mini_recovery_corruption("metadata extent entry is invalid", meta_addr); } - /* - * Report a metadata extent before reading it. The normal mini metadata - * layout is contiguous within an extent, so this is once per physical - * metadata extent; next-link and reciprocal-prev validation below reject - * a malformed chain before it can complete a loop. - */ - uint64 meta_extent = mini_recovery_extent_base_addr(cfg, meta_addr); - if (!mini_recovery_valid_extent_addr(cfg, meta_extent)) { + uint64 extent_addr = extent_number * state->cfg->io_cfg->extent_size; + if (!mini_recovery_valid_extent_addr(state->cfg, extent_addr)) { return mini_recovery_corruption("metadata extent is out of range", - meta_extent); - } - if (meta_extent != prior_meta_extent) { - platform_status rc = visit(meta_extent, - meta_type, - MINI_RECOVERY_EXTENT_METADATA, - MINI_RECOVERY_METADATA_BATCH, - arg); - if (!SUCCESS(rc)) { - return rc; - } - prior_meta_extent = meta_extent; - } - - page_handle *meta_page = cache_get(cc, meta_addr, TRUE, meta_type); - if (meta_page == NULL) { - return STATUS_IO_ERROR; - } - - mini_meta_hdr *hdr = (mini_meta_hdr *)meta_page->data; - platform_status rc = mini_recovery_validate_meta_header( - hdr, cfg->io_cfg->page_size, expected_prev, meta_addr); - if (!SUCCESS(rc)) { - cache_unget(cc, meta_page); - return rc; + extent_addr); } - uint64 next_meta_addr = hdr->next_meta_addr; - rc = - mini_recovery_validate_next_meta_addr(cfg, meta_addr, next_meta_addr); + rc = allocator_recovery_record_reference( + state->al, extent_addr, extent_type); if (!SUCCESS(rc)) { - cache_unget(cc, meta_page); return rc; } - meta_entry *entry = first_entry(meta_page); - for (uint64 entry_no = 0; entry_no < hdr->num_entries; entry_no++) { - uint64 batch = meta_entry_batch(entry); - page_type extent_type = meta_entry_type(entry); - uint64 extent_number = - entry->packed >> (META_ENTRY_BATCH_BITS + META_ENTRY_TYPE_BITS); - - if (batch >= MINI_MAX_BATCHES || extent_type < PAGE_TYPE_FIRST - || extent_type >= NUM_PAGE_TYPES || extent_number == 0 - || extent_number > UINT64_MAX / cfg->io_cfg->extent_size) - { - cache_unget(cc, meta_page); - return mini_recovery_corruption("metadata extent entry is invalid", - meta_addr); - } - - uint64 extent_addr = extent_number * cfg->io_cfg->extent_size; - if (!mini_recovery_valid_extent_addr(cfg, extent_addr)) { - cache_unget(cc, meta_page); - return mini_recovery_corruption("metadata extent is out of range", - extent_addr); - } + entry = next_entry(entry); + } - rc = visit( - extent_addr, extent_type, MINI_RECOVERY_EXTENT_DATA, batch, arg); - if (!SUCCESS(rc)) { - cache_unget(cc, meta_page); - return rc; - } + state->expected_prev = meta_addr; + return STATUS_OK; +} - entry = next_entry(entry); - } +/* + * mini_recover_allocations -- + * + * Rebuild allocator references for a finalized mini allocator without + * trusting allocator refcounts -- the discovery primitive crash recovery + * uses to reconstruct them. Records one reference for every metadata + * extent and every data-extent entry found in the on-disk mini metadata + * stream (data entries are not deduplicated: repeated entries mean + * repeated references, and are recorded exactly as often as they occur). + * + * meta_type is the page type of meta_head's own chain; each data extent's + * type is read from its own metadata entry. + * + * Validates the metadata-page chain, page-header bounds, page types, + * batches, and extent addresses before using them. A malformed on-disk + * stream returns STATUS_INVALID_STATE. This function performs no writes + * and does not use allocator refcounts to decide what to traverse. + * + * This is a physical enumeration only. Recovering logical reference + * multiplicity, and deduplicating references shared by distinct mini + * allocator roots, remains the responsibility of the higher-level + * trunk/log recovery walker. + */ +platform_status +mini_recover_allocations(cache *cc, uint64 meta_head, page_type meta_type) +{ + if (cc == NULL || meta_type < PAGE_TYPE_FIRST || meta_type >= NUM_PAGE_TYPES) + { + return STATUS_BAD_PARAM; + } - cache_unget(cc, meta_page); - expected_prev = meta_addr; - meta_addr = next_meta_addr; + allocator *al = cache_get_allocator(cc); + if (al == NULL) { + return STATUS_BAD_PARAM; + } + allocator_config *cfg = allocator_get_config(al); + if (!mini_recovery_valid_geometry(cfg)) { + return STATUS_BAD_PARAM; + } + if (!mini_recovery_valid_page_addr(cfg, meta_head)) { + return mini_recovery_corruption("metadata head is not a page", meta_head); } - return STATUS_OK; + mini_recover_state state = {.al = al, + .cfg = cfg, + .expected_prev = 0, + .page_count = 0, + .max_meta_pages = cfg->capacity + / cfg->io_cfg->page_size}; + + return mini_for_each_meta_page(cc, + meta_head, + meta_type, + mini_recover_visit_meta_extent, + mini_recover_visit_meta_page, + &state); } /* @@ -1314,7 +1387,7 @@ space_use_add_extent(cache *cc, page_type type, uint64 extent_addr, void *out) *sum += cache_extent_size(cc); } -static void +static platform_status space_use_add_meta_page(cache *cc, page_type type, page_handle *meta_page, @@ -1322,6 +1395,7 @@ space_use_add_meta_page(cache *cc, { uint64 *sum = (uint64 *)out; *sum += cache_page_size(cc); + return STATUS_OK; } uint64 @@ -1330,7 +1404,7 @@ mini_space_use_bytes(cache *cc, uint64 meta_head, page_type type) uint64 total = 0; mini_for_each(cc, meta_head, type, space_use_add_extent, &total); mini_for_each_meta_page( - cc, meta_head, type, space_use_add_meta_page, &total); + cc, meta_head, type, NULL, space_use_add_meta_page, &total); return total; } diff --git a/src/mini_allocator.h b/src/mini_allocator.h index 8a655679..5407530f 100644 --- a/src/mini_allocator.h +++ b/src/mini_allocator.h @@ -114,55 +114,31 @@ void mini_prefetch(cache *cc, page_type type, uint64 meta_head); /* - * mini_recovery_walk -- + * mini_recover_allocations -- * - * Enumerate the physical extents reachable from a finalized mini - * allocator without looking at allocator refcounts. This is the - * discovery primitive used by crash recovery to rebuild those refcounts. + * Rebuild allocator references for a finalized mini allocator without + * trusting allocator refcounts. This is the discovery primitive crash + * recovery uses to reconstruct them: it records one allocator reference + * (via allocator_recovery_record_reference()) for every metadata extent + * and for every data-extent entry in the on-disk mini metadata stream. + * Data entries are deliberately not deduplicated: repeated entries + * represent repeated references in the mini allocator and are recorded + * exactly as often as they occur. meta_type is the page type of + * meta_head's own chain; each data extent's type is read from its own + * metadata entry, so it is not a parameter here. * - * The callback is invoked once for each metadata extent and once for - * every data-extent entry in the on-disk mini metadata stream. Data - * entries are deliberately not deduplicated: repeated entries represent - * repeated references in the mini allocator and are reported exactly as - * recorded. Metadata entries have batch - * MINI_RECOVERY_METADATA_BATCH. - * - * The walker validates the metadata-page chain, page-header bounds, page - * types, batches, and extent addresses before using them. A malformed - * on-disk stream returns STATUS_INVALID_STATE. Callback failures are - * returned unchanged. The walker performs no writes and does not use - * allocator refcounts to decide what to traverse. - * - * The callback is called for a metadata extent before the walker reads a - * page from that extent. That ordering lets an allocator-recovery caller - * establish temporary ownership before cache_get() performs its debug - * allocation checks. + * Validates the metadata-page chain, page-header bounds, page types, + * batches, and extent addresses before using them. A malformed on-disk + * stream returns STATUS_INVALID_STATE. This function performs no writes + * and does not use allocator refcounts to decide what to traverse. * * This is a physical enumeration only. Recovering logical reference * multiplicity, and deduplicating references shared by distinct mini * allocator roots, remains the responsibility of the higher-level * trunk/log recovery walker. */ -typedef enum mini_recovery_extent_kind { - MINI_RECOVERY_EXTENT_METADATA, - MINI_RECOVERY_EXTENT_DATA, -} mini_recovery_extent_kind; - -#define MINI_RECOVERY_METADATA_BATCH ((uint64) - 1) - -typedef platform_status (*mini_recovery_visit_fn)( - uint64 extent_addr, - page_type type, - mini_recovery_extent_kind kind, - uint64 batch, - void *arg); - platform_status -mini_recovery_walk(cache *cc, - uint64 meta_head, - page_type meta_type, - mini_recovery_visit_fn visit, - void *arg); +mini_recover_allocations(cache *cc, uint64 meta_head, page_type meta_type); /* * mini_meta_cursor: a non-blocking cursor over the extent entries of a diff --git a/src/rc_allocator.c b/src/rc_allocator.c index f11a4524..e86d8519 100644 --- a/src/rc_allocator.c +++ b/src/rc_allocator.c @@ -123,6 +123,20 @@ rc_allocator_get_ref_virtual(allocator *a, uint64 addr) return rc_allocator_get_ref(al, addr); } +platform_status +rc_allocator_rebuild_acquire_extent(rc_allocator *al, + uint64 extent_addr, + page_type type); + +platform_status +rc_allocator_recovery_record_reference_virtual(allocator *a, + uint64 addr, + page_type type) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_rebuild_acquire_extent(al, addr, type); +} + platform_status rc_allocator_get_super_addr(rc_allocator *al, allocator_root_id spl_id, @@ -217,6 +231,7 @@ const static allocator_ops rc_allocator_ops = { .inc_ref = rc_allocator_inc_ref_virtual, .dec_ref = rc_allocator_dec_ref_virtual, .get_ref = rc_allocator_get_ref_virtual, + .recovery_record_reference = rc_allocator_recovery_record_reference_virtual, .get_super_addr = rc_allocator_get_super_addr_virtual, .alloc_super_addr = rc_allocator_alloc_super_addr_virtual, .remove_super_addr = rc_allocator_remove_super_addr_virtual, @@ -940,7 +955,9 @@ rc_allocator_mount_recovery(rc_allocator *al, } platform_status -rc_allocator_rebuild_acquire_extent(rc_allocator *al, uint64 extent_addr) +rc_allocator_rebuild_acquire_extent(rc_allocator *al, + uint64 extent_addr, + page_type type) { if (!al->recovery_in_progress) { platform_error_log("Cannot acquire allocator extent while recovery is " @@ -982,7 +999,9 @@ rc_allocator_rebuild_acquire_extent(rc_allocator *al, uint64 extent_addr) } if (old_ref == AL_FREE) { + platform_assert(type != PAGE_TYPE_INVALID); rc_allocator_record_allocated_extent(al); + __sync_add_and_fetch(&al->stats.extent_allocs[type], 1); } return STATUS_OK; } diff --git a/src/rc_allocator.h b/src/rc_allocator.h index 28f8226c..b7ada5cc 100644 --- a/src/rc_allocator.h +++ b/src/rc_allocator.h @@ -136,11 +136,14 @@ rc_allocator_mount_recovery(rc_allocator *al, /* * Add one logical ownership reference for an extent while rebuilding a * recovery map. The first reference establishes the allocator's nonzero - * allocation floor (AL_ONE_REF); later references increment it normally. - * extent_addr must be the base address of a non-reserved allocator extent. + * allocation floor (AL_ONE_REF) and records it in stats.extent_allocs[type]; + * later references increment the refcount normally. extent_addr must be the + * base address of a non-reserved allocator extent. */ platform_status -rc_allocator_rebuild_acquire_extent(rc_allocator *al, uint64 extent_addr); +rc_allocator_rebuild_acquire_extent(rc_allocator *al, + uint64 extent_addr, + page_type type); /* * Complete a successful rebuild without performing I/O. A later normal diff --git a/tests/functional/btree_test.c b/tests/functional/btree_test.c index 6d4e6d7a..560e4c24 100644 --- a/tests/functional/btree_test.c +++ b/tests/functional/btree_test.c @@ -162,107 +162,6 @@ test_memtable_generation_init(cache *cc, return rc; } -typedef struct test_mini_recovery_walk_state { - uint64 metadata_visits; - uint64 data_visits; - uint64 metadata_extent; - uint64 data_extent; - bool32 fail_data_visit; -} test_mini_recovery_walk_state; - -static platform_status -test_mini_recovery_walk_visit(uint64 extent_addr, - page_type type, - mini_recovery_extent_kind kind, - uint64 batch, - void *arg) -{ - test_mini_recovery_walk_state *state = arg; - - if (type != PAGE_TYPE_MISC) { - return STATUS_TEST_FAILED; - } - - if (kind == MINI_RECOVERY_EXTENT_METADATA) { - if (batch != MINI_RECOVERY_METADATA_BATCH) { - return STATUS_TEST_FAILED; - } - state->metadata_visits++; - state->metadata_extent = extent_addr; - return STATUS_OK; - } - - if (kind != MINI_RECOVERY_EXTENT_DATA || batch != 0) { - return STATUS_TEST_FAILED; - } - - state->data_visits++; - state->data_extent = extent_addr; - return state->fail_data_visit ? STATUS_TEST_FAILED : STATUS_OK; -} - -static platform_status -test_mini_recovery_walk(cache *cc) -{ - allocator *al = cache_get_allocator(cc); - mini_allocator mini; - test_mini_recovery_walk_state state; - uint64 meta_head = 0; - uint64 data_extent = 0; - platform_status rc = allocator_alloc(al, &meta_head, PAGE_TYPE_MISC); - if (!SUCCESS(rc)) { - return rc; - } - - mini_init(&mini, cc, meta_head, 0, 1, PAGE_TYPE_MISC); - data_extent = mini_alloc_extent(&mini, 0, NULL); - if (data_extent == 0) { - rc = STATUS_NO_SPACE; - goto cleanup; - } - - ZERO_CONTENTS(&state); - rc = mini_recovery_walk( - cc, meta_head, PAGE_TYPE_MISC, test_mini_recovery_walk_visit, &state); - if (!SUCCESS(rc)) { - goto cleanup; - } - if (state.metadata_visits != 1 || state.data_visits != 1 - || state.metadata_extent != meta_head - || state.data_extent != data_extent) - { - platform_error_log("mini recovery walker returned unexpected extents\n"); - rc = STATUS_TEST_FAILED; - goto cleanup; - } - - ZERO_CONTENTS(&state); - state.fail_data_visit = TRUE; - rc = mini_recovery_walk( - cc, meta_head, PAGE_TYPE_MISC, test_mini_recovery_walk_visit, &state); - if (!STATUS_IS_EQ(rc, STATUS_TEST_FAILED) || state.metadata_visits != 1 - || state.data_visits != 1) - { - platform_error_log("mini recovery walker did not propagate callback " - "failure\n"); - rc = STATUS_TEST_FAILED; - goto cleanup; - } - rc = STATUS_OK; - -cleanup: - mini_release(&mini); - refcount ref = mini_dec_ref(cc, meta_head, PAGE_TYPE_MISC); - if (ref != 0 && SUCCESS(rc)) { - platform_error_log("mini recovery walker left an unexpected mini ref\n"); - rc = STATUS_TEST_FAILED; - } - if (SUCCESS(rc)) { - platform_default_log("btree_test: mini recovery walker test passed\n"); - } - return rc; -} - test_memtable_context * test_memtable_context_create(cache *cc, test_btree_config *cfg, @@ -2454,9 +2353,6 @@ btree_test(int argc, char *argv[]) rc = test_memtable_generation_init(ccp, &test_cfg, hid); platform_assert_status_ok(rc); - rc = test_mini_recovery_walk(ccp); - platform_assert_status_ok(rc); - uint64 max_tuples_per_memtable = test_cfg.mt_cfg->max_extents_per_memtable * cache_config_extent_size((cache_config *)&system_cfg.cache_cfg) / 3 diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index 8941fe2a..6dd55db3 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -12,6 +12,7 @@ #include "rc_allocator.h" #include "cache.h" #include "clockcache.h" +#include "mini_allocator.h" #include "task.h" #include "random.h" #include "test_common.h" @@ -312,7 +313,8 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, goto cleanup; } - rc = rc_allocator_rebuild_acquire_extent(&recovery, stale_extent_addr); + rc = rc_allocator_rebuild_acquire_extent( + &recovery, stale_extent_addr, PAGE_TYPE_MISC); if (!SUCCESS(rc) || allocator_get_refcount((allocator *)&recovery, stale_extent_addr) != AL_ONE_REF) @@ -346,11 +348,13 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, } recovery_live = TRUE; - rc = rc_allocator_rebuild_acquire_extent(&recovery, stale_extent_addr); + rc = rc_allocator_rebuild_acquire_extent( + &recovery, stale_extent_addr, PAGE_TYPE_MISC); if (!SUCCESS(rc)) { goto cleanup; } - rc = rc_allocator_rebuild_acquire_extent(&recovery, stale_extent_addr); + rc = rc_allocator_rebuild_acquire_extent( + &recovery, stale_extent_addr, PAGE_TYPE_MISC); if (!SUCCESS(rc) || allocator_get_refcount((allocator *)&recovery, stale_extent_addr) != AL_ONE_REF + 1) @@ -400,6 +404,178 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, return rc; } +#define TEST_MINI_RECOVER_NUM_DATA_EXTENTS 3 + +/* + * Build a real mini_allocator with one metadata extent and several data + * extents, durably persist it, then simulate a crash: mount a fresh allocator + * in recovery mode (which deliberately ignores the persisted refcount table) + * and confirm mini_recover_allocations() rediscovers every extent from the + * on-disk metadata alone -- including the ordering property this whole + * mechanism exists for, since a rebuilding allocator considers all of these + * extents unallocated until mini_recover_allocations() itself establishes + * their references, ahead of the cache_get() calls it issues to read them. + */ +static platform_status +test_mini_recover_allocations(allocator_config *allocator_cfg, + clockcache_config *cache_cfg, + io_handle *io, + platform_heap_id hid) +{ + platform_status rc = STATUS_OK; + rc_allocator original, recovery; + clockcache original_cc, recovery_cc; + bool32 original_al_live = FALSE, original_cc_live = FALSE; + bool32 recovery_al_live = FALSE, recovery_cc_live = FALSE; + mini_allocator mini; + bool32 mini_live = FALSE; + uint64 meta_head = 0; + uint64 data_extents[TEST_MINI_RECOVER_NUM_DATA_EXTENTS] = {0}; + + platform_default_log( + "cache_test: mini_recover_allocations test started\n"); + + rc = rc_allocator_init( + &original, allocator_cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + original_al_live = TRUE; + + rc = clockcache_init(&original_cc, + cache_cfg, + io, + (allocator *)&original, + "test", + hid, + platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + original_cc_live = TRUE; + cache *original_ccp = (cache *)&original_cc; + + rc = allocator_alloc((allocator *)&original, &meta_head, PAGE_TYPE_MISC); + if (!SUCCESS(rc)) { + goto cleanup; + } + mini_init(&mini, original_ccp, meta_head, 0, 1, PAGE_TYPE_MISC); + mini_live = TRUE; + + for (uint64 i = 0; i < TEST_MINI_RECOVER_NUM_DATA_EXTENTS; i++) { + data_extents[i] = mini_alloc_extent(&mini, 0, NULL); + if (data_extents[i] == 0) { + rc = STATUS_NO_SPACE; + goto cleanup; + } + } + mini_release(&mini); + mini_live = FALSE; + + /* Durably persist before "crashing": recovery must see real disk state. */ + cache_flush(original_ccp); + clockcache_deinit(&original_cc); + original_cc_live = FALSE; + rc_allocator_unmount(&original); + original_al_live = FALSE; + + rc = rc_allocator_mount_recovery( + &recovery, allocator_cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + recovery_al_live = TRUE; + + /* mount_recovery deliberately ignores the persisted map: confirm that. */ + if (allocator_get_refcount((allocator *)&recovery, meta_head) != AL_FREE) { + platform_error_log( + "cache_test: recovery mount trusted a persisted refcount\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + + rc = clockcache_init(&recovery_cc, + cache_cfg, + io, + (allocator *)&recovery, + "test", + hid, + platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + recovery_cc_live = TRUE; + + /* + * Every extent below is still AL_FREE from recovery's point of view: this + * call must establish each one's reference itself, before its own + * cache_get() calls, or those cache_get()s would trip the cache's debug + * allocation check. + */ + rc = mini_recover_allocations( + (cache *)&recovery_cc, meta_head, PAGE_TYPE_MISC); + if (!SUCCESS(rc)) { + platform_error_log("cache_test: mini_recover_allocations failed: %s\n", + platform_status_to_string(rc)); + goto cleanup; + } + + /* + * Exactly one reference per extent is mini_recover_allocations()'s own + * documented contract; reproducing a mini_allocator's full live-lifetime + * refcount (which may include self-references beyond this) is explicitly + * the job of a higher-level recovery walker, not this function. + */ + if (allocator_get_refcount((allocator *)&recovery, meta_head) + != AL_ONE_REF) + { + platform_error_log( + "cache_test: mini_recover_allocations did not recover the metadata " + "extent reference\n"); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + for (uint64 i = 0; i < TEST_MINI_RECOVER_NUM_DATA_EXTENTS; i++) { + if (allocator_get_refcount((allocator *)&recovery, data_extents[i]) + != AL_ONE_REF) + { + platform_error_log("cache_test: mini_recover_allocations did not " + "recover data extent %lu\n", + i); + rc = STATUS_TEST_FAILED; + goto cleanup; + } + } + rc_allocator_rebuild_finish(&recovery); + +cleanup: + if (mini_live) { + mini_release(&mini); + } + if (recovery_cc_live) { + clockcache_deinit(&recovery_cc); + } + if (recovery_al_live) { + if (recovery.recovery_in_progress) { + rc_allocator_abort_recovery(&recovery); + } else { + rc_allocator_deinit(&recovery); + } + } + if (original_cc_live) { + clockcache_deinit(&original_cc); + } + if (original_al_live) { + rc_allocator_deinit(&original); + } + + if (SUCCESS(rc)) { + platform_default_log( + "cache_test: mini_recover_allocations test passed\n"); + } + return rc; +} + typedef struct { cache *cc; volatile bool32 finished; @@ -1812,6 +1988,12 @@ cache_test(int argc, char *argv[]) goto destroy_iohandle; } + rc = test_mini_recover_allocations( + &system_cfg.allocator_cfg, &system_cfg.cache_cfg, io, hid); + if (!SUCCESS(rc)) { + goto destroy_iohandle; + } + rc = test_init_task_system(&ts, hid, &system_cfg.task_cfg); if (!SUCCESS(rc)) { platform_error_log("Failed to init splinter state: %s\n", From 45137d8b95cdd3ca325bb0be552b4602dceb331c Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Thu, 23 Jul 2026 16:37:44 -0700 Subject: [PATCH 14/64] superblock reorg Signed-off-by: Rob Johnson --- Makefile | 6 + src/allocator.h | 58 +- src/core.c | 803 +++++++++----------------- src/core.h | 16 +- src/rc_allocator.c | 710 ++++------------------- src/rc_allocator.h | 104 +--- src/splinterdb.c | 41 +- tests/functional/cache_test.c | 274 ++++----- tests/functional/splinter_test.c | 16 + tests/functional/test_functionality.c | 12 + tests/functional/ycsb_test.c | 5 +- tests/unit/splinter_test.c | 7 + 12 files changed, 637 insertions(+), 1415 deletions(-) diff --git a/Makefile b/Makefile index 2d2dea65..0ac17aed 100644 --- a/Makefile +++ b/Makefile @@ -508,6 +508,11 @@ $(BINDIR)/$(UNITDIR)/task_system_test: $(COMMON_TESTOBJ) $(OBJDIR)/$(FUNCTIONAL_TESTSDIR)/test_async.o \ $(LIBDIR)/libsplinterdb.so +$(BINDIR)/$(UNITDIR)/superblock_test: $(COMMON_TESTOBJ) \ + $(COMMON_UNIT_TESTOBJ) \ + $(OBJDIR)/$(FUNCTIONAL_TESTSDIR)/test_async.o \ + $(LIBDIR)/libsplinterdb.so + $(BINDIR)/$(UNITDIR)/platform_apis_test: $(UTIL_SYS) \ $(COMMON_UNIT_TESTOBJ) \ $(OBJDIR)/$(TESTS_DIR)/config.o \ @@ -551,6 +556,7 @@ unit/writable_buffer_test: $(BINDIR)/$(UNITDIR)/writable_buffer_test unit/config_parse_test: $(BINDIR)/$(UNITDIR)/config_parse_test unit/limitations_test: $(BINDIR)/$(UNITDIR)/limitations_test unit/task_system_test: $(BINDIR)/$(UNITDIR)/task_system_test +unit/superblock_test: $(BINDIR)/$(UNITDIR)/superblock_test unit/splinter_shmem_test: $(BINDIR)/$(UNITDIR)/splinter_shmem_test unit/splinter_ipc_test: $(BINDIR)/$(UNITDIR)/splinter_ipc_test unit_test: $(BINDIR)/unit_test diff --git a/src/allocator.h b/src/allocator.h index 54c8b939..17fb5c87 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -138,21 +138,38 @@ typedef refcount (*generic_ref_fn)(allocator *al, uint64 addr); /* * Record one logical reference to addr while rebuilding a recovery map (see - * rc_allocator_mount_recovery()). The first reference to a given extent - * establishes its nonzero allocation floor; later references increment it - * normally. addr must be the base address of a non-reserved extent. + * allocator_open_refcounts() with rebuild == TRUE). The first reference to a + * given extent establishes its nonzero allocation floor; later references + * increment it normally. addr must be the base address of a non-reserved + * extent. */ typedef platform_status (*recovery_record_reference_fn)(allocator *al, uint64 addr, page_type type); -typedef platform_status (*get_super_addr_fn)(allocator *al, - allocator_root_id spl_id, - uint64 *addr); -typedef platform_status (*alloc_super_addr_fn)(allocator *al, - allocator_root_id spl_id, - uint64 *addr); -typedef void (*remove_super_addr_fn)(allocator *al, allocator_root_id spl_id); +/* + * Populate the in-memory refcount map of an allocator that has been attached + * (mounted) but whose map is not yet loaded. rebuild == FALSE loads the + * trusted persisted map from its fixed durable location; rebuild == TRUE + * initializes an empty map that reserves only the allocator's own fixed extents + * and enters recovery mode (the caller then rebuilds via + * recovery_record_reference()). Must be called exactly once, after the attach + * mount and before any allocation. The caller learns which mode to request + * from durable metadata it owns (the superblock's allocation-state validity), + * so the allocator never reads the persisted map when it would be discarded. + */ +typedef platform_status (*open_refcounts_fn)(allocator *al, bool32 rebuild); + +/* + * Write the refcount map to its durable location and make it durable. Returns + * the base address of the persisted map in *state_addr; the caller records that + * as the superblock's allocation_state_addr only after this returns, so the + * "map is trustworthy" flag never becomes durable before the map itself. Not + * valid while a recovery rebuild is in progress. + */ +typedef platform_status (*persist_refcounts_fn)(allocator *al, + uint64 *state_addr); + typedef uint64 (*get_size_fn)(allocator *al); typedef uint64 (*base_addr_fn)(const allocator *al, uint64 addr); @@ -172,9 +189,8 @@ typedef struct allocator_ops { generic_ref_fn get_ref; recovery_record_reference_fn recovery_record_reference; - alloc_super_addr_fn alloc_super_addr; - get_super_addr_fn get_super_addr; - remove_super_addr_fn remove_super_addr; + open_refcounts_fn open_refcounts; + persist_refcounts_fn persist; get_size_fn in_use; @@ -229,23 +245,15 @@ allocator_recovery_record_reference(allocator *al, uint64 addr, page_type type) } static inline platform_status -allocator_get_super_addr(allocator *al, allocator_root_id spl_id, uint64 *addr) +allocator_open_refcounts(allocator *al, bool32 rebuild) { - return al->ops->get_super_addr(al, spl_id, addr); + return al->ops->open_refcounts(al, rebuild); } static inline platform_status -allocator_alloc_super_addr(allocator *al, - allocator_root_id spl_id, - uint64 *addr) -{ - return al->ops->alloc_super_addr(al, spl_id, addr); -} - -static inline void -allocator_remove_super_addr(allocator *al, allocator_root_id spl_id) +allocator_persist(allocator *al, uint64 *state_addr) { - return al->ops->remove_super_addr(al, spl_id); + return al->ops->persist(al, state_addr); } static inline uint64 diff --git a/src/core.c b/src/core.c index d17af7bc..3250ede0 100644 --- a/src/core.c +++ b/src/core.c @@ -46,16 +46,6 @@ static const int64 latency_histo_buckets[LATENCYHISTO_SIZE] = { _Static_assert(CORE_NUM_MEMTABLES <= MAX_MEMTABLES, "CORE_NUM_MEMTABLES <= MAX_MEMTABLES"); -/* Checkpoint metadata has independently checksummed directory and records. */ -#define CORE_CHECKPOINT_DIRECTORY_CSUM_SEED (42) -#define CORE_CHECKPOINT_RECORD_CSUM_SEED (43) - -#define CORE_CHECKPOINT_FORMAT_VERSION (2) -#define CORE_CHECKPOINT_RECORD_COUNT (2) - -#define CORE_CHECKPOINT_DIRECTORY_MAGIC (0x534442434B505444ULL) // SDBCKPTD -#define CORE_CHECKPOINT_RECORD_MAGIC (0x534442434B505452ULL) // SDBCKPTR - static platform_status core_checkpoint_lock_init(core_handle *spl) { @@ -134,355 +124,6 @@ core_close_log_stream_if_enabled(core_handle *spl, } \ } while (0) -/* - *----------------------------------------------------------------------------- - * Checkpoint metadata: disk-resident structures. - * - * allocator_get_super_addr() identifies a fixed page in the allocator's - * bootstrap extent. That page is an immutable directory, written exactly - * once when the table is created. The directory names two independently - * allocated record extents. Checkpoint publication alternates between their - * first pages, leaving one formerly valid record untouched if a new write is - * torn. - * - * This is intentionally a new on-disk format. Do not interpret a legacy - * core_super_block as a directory: doing so would turn arbitrary old fields - * into allocator-owned addresses. Existing databases must be migrated or - * reformatted before using this checkpoint metadata format. - *----------------------------------------------------------------------------- - */ -typedef struct ONDISK core_checkpoint_directory { - uint64 magic; - uint64 format_version; - uint64 table_id; - uint64 record_addr[CORE_CHECKPOINT_RECORD_COUNT]; - checksum128 checksum; -} core_checkpoint_directory; - -typedef struct ONDISK core_checkpoint_record { - /* - * The highest memtable generation incorporated in root_addr. The boolean - * keeps the fresh-database case distinct from generation zero. - */ - uint64 incorporated_generation; - bool32 has_incorporated_generation; - uint64 root_addr; - uint64 timestamp; - uint64 sequence; - uint64 table_id; - uint32 record_slot; - bool32 checkpointed; - bool32 unmounted; - uint64 magic; - uint64 format_version; - checksum128 checksum; -} core_checkpoint_record; - -typedef struct core_checkpoint_records { - core_checkpoint_record record[CORE_CHECKPOINT_RECORD_COUNT]; - bool32 valid[CORE_CHECKPOINT_RECORD_COUNT]; - bool32 have_newest; - uint64 newest_slot; - bool32 have_newest_unmounted; - uint64 newest_unmounted_slot; -} core_checkpoint_records; - -static checksum128 -core_checkpoint_directory_checksum(const core_checkpoint_directory *directory) -{ - return platform_checksum128(directory, - offsetof(core_checkpoint_directory, checksum), - CORE_CHECKPOINT_DIRECTORY_CSUM_SEED); -} - -static checksum128 -core_checkpoint_record_checksum(const core_checkpoint_record *record) -{ - return platform_checksum128(record, - offsetof(core_checkpoint_record, checksum), - CORE_CHECKPOINT_RECORD_CSUM_SEED); -} - -static bool32 -core_checkpoint_record_addr_is_valid(core_handle *spl, uint64 addr) -{ - allocator_config *allocator_cfg = allocator_get_config(spl->al); - uint64 page_size = cache_page_size(spl->cc); - - return addr != 0 && addr % allocator_cfg->io_cfg->extent_size == 0 - && addr < allocator_cfg->capacity - && page_size <= allocator_cfg->capacity - addr; -} - -static bool32 -core_checkpoint_directory_is_valid(core_handle *spl, - const core_checkpoint_directory *directory) -{ - if (directory->magic != CORE_CHECKPOINT_DIRECTORY_MAGIC - || directory->format_version != CORE_CHECKPOINT_FORMAT_VERSION - || directory->table_id != spl->id - || !platform_checksum_is_equal( - directory->checksum, core_checkpoint_directory_checksum(directory))) - { - return FALSE; - } - - uint64 record0 = directory->record_addr[0]; - uint64 record1 = directory->record_addr[1]; - allocator_config *allocator_cfg = allocator_get_config(spl->al); - return core_checkpoint_record_addr_is_valid(spl, record0) - && core_checkpoint_record_addr_is_valid(spl, record1) - && record0 != record1 - && !allocator_config_pages_share_extent( - allocator_cfg, record0, record1); -} - -static bool32 -core_checkpoint_record_is_valid(core_handle *spl, - const core_checkpoint_record *record, - uint64 record_slot) -{ - return record->magic == CORE_CHECKPOINT_RECORD_MAGIC - && record->format_version == CORE_CHECKPOINT_FORMAT_VERSION - && record->table_id == spl->id && record->record_slot == record_slot - && record->sequence != 0 - && (record->has_incorporated_generation == FALSE - || record->has_incorporated_generation == TRUE) - && (record->has_incorporated_generation - ? record->incorporated_generation < UINT64_MAX - : record->incorporated_generation == 0) - && platform_checksum_is_equal( - record->checksum, core_checkpoint_record_checksum(record)); -} - -static void -core_write_checkpoint_page(core_handle *spl, - uint64 page_addr, - const void *contents, - uint64 contents_size) -{ - page_handle *page = - cache_get(spl->cc, page_addr, TRUE, PAGE_TYPE_SUPERBLOCK); - uint64 wait = 1; - while (!cache_try_claim(spl->cc, page)) { - cache_unget(spl->cc, page); - platform_sleep_ns(wait); - wait = wait > 1024 ? wait : 2 * wait; - page = cache_get(spl->cc, page_addr, TRUE, PAGE_TYPE_SUPERBLOCK); - } - cache_lock(spl->cc, page); - platform_assert(contents_size <= cache_page_size(spl->cc)); - memset(page->data, 0, cache_page_size(spl->cc)); - memcpy(page->data, contents, contents_size); - cache_unlock(spl->cc, page); - cache_unclaim(spl->cc, page); - cache_page_writeback(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); - cache_unget(spl->cc, page); -} - -static void -core_initialize_checkpoint_record_page(core_handle *spl, uint64 page_addr) -{ - page_handle *page = cache_alloc(spl->cc, page_addr, PAGE_TYPE_SUPERBLOCK); - platform_assert(page != NULL); - memset(page->data, 0, cache_page_size(spl->cc)); - cache_unlock(spl->cc, page); - cache_unclaim(spl->cc, page); - cache_page_writeback(spl->cc, page, TRUE, PAGE_TYPE_SUPERBLOCK); - cache_unget(spl->cc, page); -} - -static platform_status -core_create_checkpoint_directory(core_handle *spl, - core_checkpoint_directory *directory) -{ - uint64 directory_addr; - platform_status rc = - allocator_alloc_super_addr(spl->al, spl->id, &directory_addr); - if (!SUCCESS(rc)) { - platform_error_log("core_create_checkpoint_directory: failed to allocate " - "directory address for root id %lu: %s\n", - spl->id, - platform_status_to_string(rc)); - return rc; - } - - ZERO_CONTENTS(directory); - directory->magic = CORE_CHECKPOINT_DIRECTORY_MAGIC; - directory->format_version = CORE_CHECKPOINT_FORMAT_VERSION; - directory->table_id = spl->id; - - for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { - uint64 record_addr; - rc = allocator_alloc(spl->al, &record_addr, PAGE_TYPE_SUPERBLOCK); - if (!SUCCESS(rc)) { - platform_error_log("core_create_checkpoint_directory: failed to " - "allocate record extent %lu: %s\n", - slot, - platform_status_to_string(rc)); - return rc; - } - directory->record_addr[slot] = record_addr; - core_initialize_checkpoint_record_page(spl, record_addr); - } - - directory->checksum = core_checkpoint_directory_checksum(directory); - core_write_checkpoint_page( - spl, directory_addr, directory, sizeof(*directory)); - - /* - * Submit the newly initialized record and directory pages before the - * durable barrier. allocator_alloc() changes its refcount map only in - * memory; crash recovery deliberately rebuilds that map instead of relying - * on this publication. allocator_alloc_super_addr() does write the raw - * bootstrap mapping through the shared backing I/O handle, which the - * following durable barrier fdatasyncs with the directory page. - */ - rc = cache_writeback_dirty(spl->cc); - if (!SUCCESS(rc)) { - return rc; - } - return cache_durable_barrier(spl->cc); -} - -static platform_status -core_get_checkpoint_directory(core_handle *spl, - core_checkpoint_directory *directory) -{ - uint64 directory_addr; - platform_status rc = - allocator_get_super_addr(spl->al, spl->id, &directory_addr); - if (!SUCCESS(rc)) { - return rc; - } - - page_handle *page = - cache_get(spl->cc, directory_addr, TRUE, PAGE_TYPE_SUPERBLOCK); - memcpy(directory, page->data, sizeof(*directory)); - cache_unget(spl->cc, page); - - if (!core_checkpoint_directory_is_valid(spl, directory)) { - platform_error_log("core_get_checkpoint_directory: no compatible " - "checkpoint directory for root id %lu\n", - spl->id); - return STATUS_BAD_PARAM; - } - return STATUS_OK; -} - -static platform_status -core_load_checkpoint_records(core_handle *spl, - const core_checkpoint_directory *directory, - core_checkpoint_records *records) -{ - ZERO_CONTENTS(records); - for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { - page_handle *page = cache_get( - spl->cc, directory->record_addr[slot], TRUE, PAGE_TYPE_SUPERBLOCK); - memcpy(&records->record[slot], page->data, sizeof(records->record[slot])); - cache_unget(spl->cc, page); - - records->valid[slot] = - core_checkpoint_record_is_valid(spl, &records->record[slot], slot); - if (!records->valid[slot]) { - continue; - } - - if (!records->have_newest - || records->record[records->newest_slot].sequence - < records->record[slot].sequence) - { - records->have_newest = TRUE; - records->newest_slot = slot; - } - if (records->record[slot].unmounted - && (!records->have_newest_unmounted - || records->record[records->newest_unmounted_slot].sequence - < records->record[slot].sequence)) - { - records->have_newest_unmounted = TRUE; - records->newest_unmounted_slot = slot; - } - } - - if (records->valid[0] && records->valid[1] - && records->record[0].sequence == records->record[1].sequence) - { - platform_error_log("core_load_checkpoint_records: duplicate record " - "sequence %lu for root id %lu\n", - records->record[0].sequence, - spl->id); - return STATUS_BAD_PARAM; - } - return STATUS_OK; -} - -static void -core_destroy_checkpoint_record_extent(core_handle *spl, uint64 record_addr) -{ - refcount ref = allocator_dec_ref(spl->al, record_addr, PAGE_TYPE_SUPERBLOCK); - if (ref != AL_NO_REFS) { - platform_error_log("core_destroy_checkpoint_record_extent: record extent " - "%lu has unexpected refcount %u\n", - record_addr, - ref); - return; - } - - cache_extent_discard(spl->cc, record_addr, PAGE_TYPE_SUPERBLOCK); - ref = allocator_dec_ref(spl->al, record_addr, PAGE_TYPE_SUPERBLOCK); - platform_assert(ref == AL_FREE); -} - -static void -core_destroy_checkpoint_storage(core_handle *spl) -{ - core_checkpoint_directory directory; - platform_status rc = core_get_checkpoint_directory(spl, &directory); - if (!SUCCESS(rc)) { - platform_error_log("core_destroy_checkpoint_storage: unable to load " - "checkpoint directory for root id %lu: %s\n", - spl->id, - platform_status_to_string(rc)); - return; - } - - core_checkpoint_records records; - rc = core_load_checkpoint_records(spl, &directory, &records); - if (!SUCCESS(rc)) { - platform_error_log("core_destroy_checkpoint_storage: unable to load " - "checkpoint records for root id %lu: %s\n", - spl->id, - platform_status_to_string(rc)); - return; - } - - /* - * Both valid slots own independent root references. Keeping the older - * record live makes it a real fallback if the next record write is torn; - * its reference is released only when that slot is successfully - * overwritten. This is clean destruction, so release both record owners. - */ - for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { - if (!records.valid[slot] || records.record[slot].root_addr == 0) { - continue; - } - trunk_snapshot old_snapshot = {.root_addr = records.record[slot].root_addr}; - rc = trunk_snapshot_release(&spl->trunk_context, &old_snapshot); - if (!SUCCESS(rc)) { - platform_error_log("core_destroy_checkpoint_storage: failed to " - "release record %lu root %lu: %s\n", - slot, - records.record[slot].root_addr, - platform_status_to_string(rc)); - } - } - - for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { - core_destroy_checkpoint_record_extent(spl, directory.record_addr[slot]); - } -} - /* *----------------------------------------------------------------------------- * Checkpoint record functions @@ -514,27 +155,29 @@ core_capture_checkpoint_cut(core_handle *spl, return STATUS_OK; } +/* + * Publish a tree-record root advance to the superblock: capture the current + * COW root, make it durable, record it (with the incorporated-generation cut + * and the unmounted flag), invalidate the persisted allocation state in the + * same atomic update, and release the previously published root. Used by mkfs + * (empty root) and by unmount (Part A; unmount then persists the map and + * republishes a valid allocation state). The snapshot's owned reference is + * transferred to the durable record on a successful publish. + */ static platform_status -core_publish_checkpoint_record(core_handle *spl, - bool32 is_checkpoint, - bool32 is_unmount, - bool32 is_create) -{ - uint64 old_root_addr; - platform_status rc; - trunk_snapshot snapshot; - bool32 has_incorporated_generation; - uint64 incorporated_generation; - core_checkpoint_directory directory; - core_checkpoint_records records; - uint64 target_slot; - core_checkpoint_record record; +core_publish_root_record(core_handle *spl, bool32 is_unmount) +{ + platform_status rc; + trunk_snapshot snapshot; + bool32 has_incorporated_generation; + uint64 incorporated_generation; + uint64 old_root_addr = 0; + superblock_tree_record old_rec; + superblock_tree_record rec; /* - * The snapshot, target-slot selection, durable record write, and old-slot - * release are one publication transaction. In particular, two concurrent - * publishers must never choose the same target slot or release the same - * former record owner. + * The snapshot cut, durable record write, and old-root release are one + * publication transaction; serialize against any concurrent publisher. */ rc = platform_mutex_lock(&spl->checkpoint_lock); if (!SUCCESS(rc)) { @@ -549,10 +192,10 @@ core_publish_checkpoint_record(core_handle *spl, /* * The snapshot reference makes the root stable, but not necessarily - * durable. Drain only the cache intervals that existed at this cut before - * making a record that can name the root durable. This can incidentally - * persist newer log/data pages, but it does not seal or publish a logical - * durable-log tail; tail sync is a separate operation. + * durable. Drain the cache so the pages the record will name are durable + * before we publish a superblock that points at them. This can + * incidentally persist newer log/data pages, but it does not seal or + * publish a logical durable-log tail; tail sync is a separate operation. */ rc = cache_writeback_dirty(spl->cc); if (!SUCCESS(rc)) { @@ -563,84 +206,77 @@ core_publish_checkpoint_record(core_handle *spl, goto release_snapshot; } - if (is_create) { - rc = core_create_checkpoint_directory(spl, &directory); - } else { - rc = core_get_checkpoint_directory(spl, &directory); - } - if (!SUCCESS(rc)) { - platform_error_log("core_publish_checkpoint_record: failed to %s " - "checkpoint directory for root id %lu: %s\n", - is_create ? "create" : "load", - spl->id, - platform_status_to_string(rc)); - goto release_snapshot; + // The previously published root, retained until the new one is durable. + if (SUCCESS(superblock_get_tree_record(&spl->superblock, spl->id, &old_rec))) + { + old_root_addr = old_rec.root_addr; } - rc = core_load_checkpoint_records(spl, &directory, &records); - if (!SUCCESS(rc)) { - goto release_snapshot; - } + ZERO_CONTENTS(&rec); + rec.table_id = spl->id; + rec.root_addr = snapshot.root_addr; + rec.log_meta_head = 0; // set once the per-tree log is wired + rec.incorporated_generation = + has_incorporated_generation ? incorporated_generation + : SUPERBLOCK_NO_INCORPORATED_GENERATION; + rec.unmounted = is_unmount; - if (records.have_newest - && records.record[records.newest_slot].sequence == UINT64_MAX) - { - rc = STATUS_LIMIT_EXCEEDED; + rc = superblock_set_tree_record(&spl->superblock, &rec); + if (!SUCCESS(rc)) { goto release_snapshot; } - target_slot = records.have_newest ? records.newest_slot ^ 1 : 0; /* - * Each valid record owns its root reference. This publication overwrites - * target_slot, so retain that slot's old root until the replacement page - * is durable, then release only the overwritten owner. The newest record - * remains independently live as the torn-write fallback. + * Advancing a root invalidates the persisted allocation map: the in-memory + * map now diverges from disk. A clean unmount republishes a valid + * allocation state only after persisting the map (Part B). */ - old_root_addr = - records.valid[target_slot] ? records.record[target_slot].root_addr : 0; - - ZERO_CONTENTS(&record); - record.incorporated_generation = incorporated_generation; - record.has_incorporated_generation = has_incorporated_generation; - record.root_addr = snapshot.root_addr; - record.timestamp = platform_get_real_time(); - record.sequence = records.have_newest - ? records.record[records.newest_slot].sequence + 1 - : 1; - record.table_id = spl->id; - record.record_slot = target_slot; - record.checkpointed = is_checkpoint; - record.unmounted = is_unmount; - record.magic = CORE_CHECKPOINT_RECORD_MAGIC; - record.format_version = CORE_CHECKPOINT_FORMAT_VERSION; - - record.checksum = core_checkpoint_record_checksum(&record); - - core_write_checkpoint_page( - spl, directory.record_addr[target_slot], &record, sizeof(record)); - /* The record now owns this reference, even if the barrier reports failure. - */ - snapshot.root_addr = 0; + superblock_set_allocation_state_addr(&spl->superblock, 0); - rc = cache_durable_barrier(spl->cc); + rc = superblock_publish(&spl->superblock); if (!SUCCESS(rc)) { - /* The new record may be durable, so retain its transferred root ref. */ - goto unlock_checkpoint; + /* The old root is still the newest durable one; keep its reference. */ + goto release_snapshot; } - if (old_root_addr != 0) { - trunk_snapshot old_snapshot = {.root_addr = old_root_addr}; - rc = trunk_snapshot_release(&spl->trunk_context, &old_snapshot); - if (!SUCCESS(rc)) { - platform_error_log("core_publish_checkpoint_record: " - "trunk_snapshot_release failed for old root addr " - "%lu: %s\n", - old_root_addr, - platform_status_to_string(rc)); - goto unlock_checkpoint; + if (old_root_addr == rec.root_addr) { + /* + * Republishing the same root (e.g. a clean unmount with no advance since + * the last publish): the previously published record already accounts + * for a reference to it, so drop the redundant snapshot reference we + * captured rather than letting the root's refcount grow each cycle. + */ + if (snapshot.root_addr != 0) { + platform_status release_rc = + trunk_snapshot_release(&spl->trunk_context, &snapshot); + if (SUCCESS(rc) && !SUCCESS(release_rc)) { + rc = release_rc; + } + } + } else { + /* + * New root: the snapshot reference becomes the record's durable one, and + * the previously published root's reference is released. The publish + * barrier committed the new root to the newer superblock slot, so the + * old slot is no longer the mount choice and releasing it cannot strand a + * torn-write fallback. + */ + snapshot.root_addr = 0; // transferred to the durable record + if (old_root_addr != 0) { + trunk_snapshot old_snapshot = {.root_addr = old_root_addr}; + platform_status release_rc = + trunk_snapshot_release(&spl->trunk_context, &old_snapshot); + if (!SUCCESS(release_rc)) { + platform_error_log("core_publish_root_record: trunk_snapshot_release " + "failed for old root addr %lu: %s\n", + old_root_addr, + platform_status_to_string(release_rc)); + if (SUCCESS(rc)) { + rc = release_rc; + } + } } } - rc = STATUS_OK; goto unlock_checkpoint; release_snapshot: @@ -2286,6 +1922,7 @@ core_mkfs(core_handle *spl, core_config *cfg, allocator *al, cache *cc, + io_handle *io, task_system *ts, allocator_root_id id, platform_heap_id hid) @@ -2308,6 +1945,21 @@ core_mkfs(core_handle *spl, return rc; } + // Fresh superblock: geometry, empty tree table, allocation state invalid. + allocator_config *allocator_cfg = allocator_get_config(al); + rc = superblock_context_init(&spl->superblock, io, allocator_cfg, hid); + if (!SUCCESS(rc)) { + platform_error_log("core_mkfs: superblock_context_init failed: %s\n", + platform_status_to_string(rc)); + goto deinit_checkpoint_lock; + } + rc = superblock_format(&spl->superblock, allocator_cfg); + if (!SUCCESS(rc)) { + platform_error_log("core_mkfs: superblock_format failed: %s\n", + platform_status_to_string(rc)); + goto deinit_superblock; + } + // set up the memtable context memtable_config *mt_cfg = &spl->cfg.mt_cfg; rc = memtable_context_init(&spl->mt_ctxt, @@ -2319,7 +1971,7 @@ core_mkfs(core_handle *spl, if (!SUCCESS(rc)) { platform_error_log("core_mkfs: memtable_context_init failed: %s\n", platform_status_to_string(rc)); - goto deinit_checkpoint_lock; + goto deinit_superblock; } // set up the log @@ -2352,11 +2004,11 @@ core_mkfs(core_handle *spl, goto deinit_trunk_context; } - rc = core_publish_checkpoint_record(spl, FALSE, FALSE, TRUE); + // Establish the initial (empty) tree record; publish it durably. + rc = core_publish_root_record(spl, FALSE); if (!SUCCESS(rc)) { - platform_error_log( - "core_mkfs: core_publish_checkpoint_record failed: %s\n", - platform_status_to_string(rc)); + platform_error_log("core_mkfs: core_publish_root_record failed: %s\n", + platform_status_to_string(rc)); goto deinit_stats; } return STATUS_OK; @@ -2372,6 +2024,8 @@ core_mkfs(core_handle *spl, } deinit_memtable_context: memtable_context_deinit(&spl->mt_ctxt); +deinit_superblock: + superblock_context_deinit(&spl->superblock); deinit_checkpoint_lock: core_checkpoint_lock_deinit(spl); return rc; @@ -2385,6 +2039,7 @@ core_mount(core_handle *spl, core_config *cfg, allocator *al, cache *cc, + io_handle *io, task_system *ts, allocator_root_id id, platform_heap_id hid) @@ -2407,48 +2062,60 @@ core_mount(core_handle *spl, return rc; } - /* - * Preserve the historical clean-only mount rule for this first format - * slice: an interrupted run is not replayed yet, so only an explicitly - * unmounted record supplies the root. We still validate both records and - * choose the newest clean one by sequence rather than wall-clock time. - */ - uint64 root_addr = 0; - bool32 has_incorporated_generation = FALSE; - uint64 incorporated_generation = 0; - core_checkpoint_directory directory; - core_checkpoint_records records; - rc = core_get_checkpoint_directory(spl, &directory); + // Read the superblock (newest valid A/B copy; validates geometry). + allocator_config *allocator_cfg = allocator_get_config(al); + rc = superblock_context_init(&spl->superblock, io, allocator_cfg, hid); if (!SUCCESS(rc)) { + platform_error_log("core_mount: superblock_context_init failed: %s\n", + platform_status_to_string(rc)); goto deinit_checkpoint_lock; } - rc = core_load_checkpoint_records(spl, &directory, &records); + rc = superblock_mount(&spl->superblock, allocator_cfg); if (!SUCCESS(rc)) { - goto deinit_checkpoint_lock; + platform_error_log("core_mount: superblock_mount failed: %s\n", + platform_status_to_string(rc)); + goto deinit_superblock; } - if (!records.have_newest) { - platform_error_log("core_mount: checkpoint directory for root id %lu " - "has no valid records\n", + + superblock_tree_record rec; + rc = superblock_get_tree_record(&spl->superblock, spl->id, &rec); + if (!SUCCESS(rc)) { + platform_error_log("core_mount: no tree record for root id %lu\n", spl->id); - rc = STATUS_BAD_PARAM; - goto deinit_checkpoint_lock; + goto deinit_superblock; } - const core_checkpoint_record *record = &records.record[records.newest_slot]; - if (!record->unmounted) { - /* - * This is an interrupted run. An older clean record is only an A/B - * torn-write fallback, not permission to silently discard the newer - * checkpoint and its log suffix. Do not overwrite its metadata before - * log replay and allocator reconstruction are wired. - */ + + /* + * Preserve the historical clean-only mount rule for this first format + * slice: crash recovery (log replay + allocator rebuild) is not wired yet, + * so only a clean unmount -- one whose tree record is flagged unmounted and + * whose allocation state is still valid -- may supply the root. + */ + bool32 rebuild = !superblock_allocation_state_valid(&spl->superblock); + if (rebuild || !rec.unmounted) { platform_error_log("core_mount: root id %lu requires crash recovery\n", spl->id); rc = STATUS_INVALID_STATE; - goto deinit_checkpoint_lock; + goto deinit_superblock; } - root_addr = record->root_addr; - has_incorporated_generation = record->has_incorporated_generation; - incorporated_generation = record->incorporated_generation; + + /* + * Load the trusted refcount map (rebuild == FALSE on this path). This must + * precede trunk_snapshot_create_from_addr(), which increments the root's + * refcount in the now-loaded map. + */ + rc = allocator_open_refcounts(al, rebuild); + if (!SUCCESS(rc)) { + platform_error_log("core_mount: allocator_open_refcounts failed: %s\n", + platform_status_to_string(rc)); + goto deinit_superblock; + } + + uint64 root_addr = rec.root_addr; + uint64 resume_generation = + rec.incorporated_generation == SUPERBLOCK_NO_INCORPORATED_GENERATION + ? 0 + : rec.incorporated_generation + 1; memtable_config *mt_cfg = &spl->cfg.mt_cfg; rc = memtable_context_init_at_generation( @@ -2458,12 +2125,12 @@ core_mount(core_handle *spl, mt_cfg, core_memtable_flush_virtual, spl, - has_incorporated_generation ? incorporated_generation + 1 : 0); + resume_generation); if (!SUCCESS(rc)) { platform_error_log("core_mount: memtable_context_init_at_generation " "failed: %s\n", platform_status_to_string(rc)); - goto deinit_checkpoint_lock; + goto deinit_superblock; } if (spl->cfg.use_log) { @@ -2504,11 +2171,24 @@ core_mount(core_handle *spl, goto deinit_trunk_context; } - rc = core_publish_checkpoint_record(spl, FALSE, FALSE, FALSE); + /* + * Mark dirty: flip the tree record to not-unmounted and invalidate the + * persisted allocation state, atomically, before any allocation diverges + * the in-memory map from disk. A crash after this forces the next mount + * into recovery instead of silently reverting to this now-stale root. This + * keeps the same root (no snapshot capture, no refcount change). + */ + rec.unmounted = FALSE; + rc = superblock_set_tree_record(&spl->superblock, &rec); if (!SUCCESS(rc)) { - platform_error_log( - "core_mount: core_publish_checkpoint_record failed: %s\n", - platform_status_to_string(rc)); + goto deinit_stats; + } + superblock_set_allocation_state_addr(&spl->superblock, 0); + rc = superblock_publish(&spl->superblock); + if (!SUCCESS(rc)) { + platform_error_log("core_mount: mark-dirty superblock_publish failed: " + "%s\n", + platform_status_to_string(rc)); goto deinit_stats; } return STATUS_OK; @@ -2524,6 +2204,8 @@ core_mount(core_handle *spl, } deinit_memtable_context: memtable_context_deinit(&spl->mt_ctxt); +deinit_superblock: + superblock_context_deinit(&spl->superblock); deinit_checkpoint_lock: core_checkpoint_lock_deinit(spl); return rc; @@ -2631,18 +2313,52 @@ core_unmount(core_handle *spl) /* * Quiescing leaves the memtable and log contexts live so publication can - * atomically capture the retired generation and root, then record log - * metadata. Teardown is safe regardless of publication success. + * atomically capture the retired generation and root. Teardown is safe + * regardless of publication success. */ core_quiesce_for_shutdown(spl); - rc = core_publish_checkpoint_record(spl, FALSE, TRUE, FALSE); + + /* + * Part A: publish the clean-unmount root. Allocation state stays invalid + * here; it becomes valid only in Part B, after the map is persisted. + */ + rc = core_publish_root_record(spl, TRUE); if (!SUCCESS(rc)) { - platform_error_log( - "core_unmount: failed to publish checkpoint record: %s\n", - platform_status_to_string(rc)); + platform_error_log("core_unmount: failed to publish unmount root: %s\n", + platform_status_to_string(rc)); } + core_teardown_after_shutdown(spl); + /* + * Release the context's live root reference before persisting the map, so + * the persisted refcounts reflect exactly the durable record's single + * reference to the root. + */ trunk_context_deinit(&spl->trunk_context); + + /* + * Part B: only after a clean Part A publish, persist the refcount map and + * republish a valid allocation state pointing at it. The map is made + * durable before the "map is trustworthy" flag, and that flag becomes + * durable only after the root it must agree with (Part A). On a Part A + * failure we leave the allocation state invalid so the next open rebuilds. + */ + if (SUCCESS(rc)) { + uint64 map_addr; + platform_status prc = allocator_persist(spl->al, &map_addr); + if (SUCCESS(prc)) { + superblock_set_allocation_state_addr(&spl->superblock, map_addr); + prc = superblock_publish(&spl->superblock); + } + if (!SUCCESS(prc)) { + platform_error_log("core_unmount: failed to publish clean allocation " + "state: %s\n", + platform_status_to_string(prc)); + rc = prc; + } + } + + superblock_context_deinit(&spl->superblock); core_destroy_stats(spl); core_checkpoint_lock_deinit(spl); return rc; @@ -2655,13 +2371,46 @@ void core_destroy(core_handle *spl) { core_quiesce_for_shutdown(spl); + + /* + * Release the reference the published tree record holds on its root before + * tearing down the context (which releases the context's own live root + * reference). Together these free the whole tree. Release must precede + * trunk_context_deinit(): trunk_snapshot_release() needs the live context. + */ + superblock_tree_record rec; + if (SUCCESS(superblock_get_tree_record(&spl->superblock, spl->id, &rec)) + && rec.root_addr != 0) + { + trunk_snapshot old_snapshot = {.root_addr = rec.root_addr}; + platform_status rc = + trunk_snapshot_release(&spl->trunk_context, &old_snapshot); + if (!SUCCESS(rc)) { + platform_error_log( + "core_destroy: failed to release root addr %lu: %s\n", + rec.root_addr, + platform_status_to_string(rc)); + } + } + core_teardown_after_shutdown(spl); - /* Records own trunk references and their two dedicated record extents. */ - core_destroy_checkpoint_storage(spl); trunk_context_deinit(&spl->trunk_context); - // clear out this splinter table from the meta page. - allocator_remove_super_addr(spl->al, spl->id); + // Erase this tree's record and persist a clean, valid allocation state. + superblock_remove_tree_record(&spl->superblock, spl->id); + uint64 map_addr; + platform_status prc = allocator_persist(spl->al, &map_addr); + if (SUCCESS(prc)) { + superblock_set_allocation_state_addr(&spl->superblock, map_addr); + prc = superblock_publish(&spl->superblock); + } + if (!SUCCESS(prc)) { + platform_error_log("core_destroy: failed to persist allocation state: " + "%s\n", + platform_status_to_string(prc)); + } + + superblock_context_deinit(&spl->superblock); core_destroy_stats(spl); core_checkpoint_lock_deinit(spl); } @@ -2696,58 +2445,36 @@ core_print_space_use(platform_log_handle *log_handle, core_handle *spl) /* * core_print_super_block() * - * Print the fixed checkpoint directory and both independently written record - * slots for a running Splinter instance. + * Print this instance's superblock tree record and the persisted allocation + * state. */ void core_print_super_block(platform_log_handle *log_handle, core_handle *spl) { - core_checkpoint_directory directory; - platform_status rc = core_get_checkpoint_directory(spl, &directory); + superblock_tree_record rec; + platform_status rc = + superblock_get_tree_record(&spl->superblock, spl->id, &rec); if (!SUCCESS(rc)) { platform_log(log_handle, - "No compatible checkpoint directory for root id %lu\n", + "No superblock tree record for root id %lu\n", spl->id); return; } - core_checkpoint_records records; - rc = core_load_checkpoint_records(spl, &directory, &records); - if (!SUCCESS(rc)) { - platform_log(log_handle, - "Unable to load checkpoint records for root id %lu: %s\n", - spl->id, - platform_status_to_string(rc)); - return; - } - platform_log(log_handle, - "Checkpoint directory root_id=%lu record_addr=[%lu, %lu] {\n", - directory.table_id, - directory.record_addr[0], - directory.record_addr[1]); - for (uint64 slot = 0; slot < CORE_CHECKPOINT_RECORD_COUNT; slot++) { - if (!records.valid[slot]) { - platform_log(log_handle, " record[%lu]: invalid\n", slot); - continue; - } - core_checkpoint_record *record = &records.record[slot]; - platform_log(log_handle, - " record[%lu]: sequence=%lu root_addr=%lu " - "has_incorporated_generation=%d " - "incorporated_generation=%lu " - "timestamp=%lu " - "checkpointed=%d unmounted=%d\n", - slot, - record->sequence, - record->root_addr, - record->has_incorporated_generation, - record->incorporated_generation, - record->timestamp, - record->checkpointed, - record->unmounted); - } - platform_log(log_handle, "}\n\n"); + "Superblock tree record root_id=%lu {\n" + " root_addr=%lu log_meta_head=%lu\n" + " incorporated_generation=%lu unmounted=%d\n" + " allocation_state: %s (addr=%lu)\n" + "}\n\n", + rec.table_id, + rec.root_addr, + rec.log_meta_head, + rec.incorporated_generation, + rec.unmounted, + superblock_allocation_state_valid(&spl->superblock) ? "valid" + : "invalid", + superblock_allocation_state_addr(&spl->superblock)); } // clang-format off diff --git a/src/core.h b/src/core.h index 997c0622..440a2aef 100644 --- a/src/core.h +++ b/src/core.h @@ -14,6 +14,7 @@ #include "log.h" #include "trunk.h" #include "histogram.h" +#include "superblock.h" /* * Upper-bound on most number of branches that we can find our lookup-key in. @@ -109,7 +110,6 @@ struct core_handle { core_config cfg; platform_heap_id heap_id; - uint64 super_block_idx; allocator_root_id id; allocator *al; @@ -119,7 +119,17 @@ struct core_handle { trunk_context trunk_context; memtable_context mt_ctxt; - /* Serializes snapshot cuts and A/B checkpoint-record publication. */ + /* + * Durable instance metadata. core owns the in-memory superblock context + * (allocated at mkfs/mount, torn down at unmount/destroy): it borrows the + * geometry, reads its tree record, and publishes root advances plus + * allocation-state transitions. For now the instance holds a single tree; + * when multi-tree support lands this ownership hoists to an instance level + * that per-tree cores borrow. + */ + superblock_context superblock; + + /* Serializes snapshot cuts and superblock publication. */ platform_mutex checkpoint_lock; bool32 checkpoint_lock_initialized; @@ -227,6 +237,7 @@ core_mkfs(core_handle *spl, core_config *cfg, allocator *al, cache *cc, + io_handle *io, task_system *ts, allocator_root_id id, platform_heap_id hid); @@ -236,6 +247,7 @@ core_mount(core_handle *spl, core_config *cfg, allocator *al, cache *cc, + io_handle *io, task_system *ts, allocator_root_id id, platform_heap_id hid); diff --git a/src/rc_allocator.c b/src/rc_allocator.c index e86d8519..dfb62236 100644 --- a/src/rc_allocator.c +++ b/src/rc_allocator.c @@ -17,21 +17,17 @@ #include "platform_typed_alloc.h" #include "poison.h" -#define RC_ALLOCATOR_META_PAGE_CSUM_SEED (2718281828) -#define RC_ALLOCATOR_CLEAN_STATE_CSUM_SEED (2718281829) - -#define RC_ALLOCATOR_FORMAT_MAGIC (0x534442414C4C4F43ULL) // SDBALLOC -#define RC_ALLOCATOR_FORMAT_VERSION (1) - -#define RC_ALLOCATOR_CLEAN_STATE_MAGIC (0x534442434C45414EULL) // SDBCLEAN -#define RC_ALLOCATOR_CLEAN_STATE_VERSION (1) -#define RC_ALLOCATOR_CLEAN_STATE_SLOTS (2) - /* - * Base offset from where the allocator starts. Currently hard coded to 0. + * The refcount map is the allocator's only durable structure. It lives in a + * fixed run of extents starting at extent 1 (extent 0 is reserved for the + * superblock). Whether the persisted map is trustworthy is recorded by the + * superblock (allocation_state_addr), which the caller consults to choose a + * clean vs recovery mount -- the allocator itself no longer keeps a + * clean-state record or a bootstrap meta page. */ -#define RC_ALLOCATOR_BASE_OFFSET (0) +/* Extent 0 holds the superblock; the refcount map begins at extent 1. */ +#define RC_ALLOCATOR_REFCOUNT_MAP_EXTENT (1) /* A predicate defining whether to trace allocations/ref-count changes * on a given address. @@ -43,28 +39,6 @@ */ #define SHOULD_TRACE(addr) (0) // Do not trace anything -/* - * A/B clean-state records live in two fixed extents after the refcount map. - * They are deliberately separate from the sole allocator bootstrap page: a - * torn state update must leave an older valid state record available for the - * next mount or rebuild. - */ -typedef struct ONDISK rc_allocator_clean_state { - uint64 magic; - uint64 format_version; - uint64 sequence; - bool32 clean_shutdown; - checksum128 checksum; -} rc_allocator_clean_state; - -typedef struct rc_allocator_clean_states { - rc_allocator_clean_state state[RC_ALLOCATOR_CLEAN_STATE_SLOTS]; - bool32 valid[RC_ALLOCATOR_CLEAN_STATE_SLOTS]; - bool32 have_newest; - uint64 newest_slot; - bool32 duplicate_sequence; -} rc_allocator_clean_states; - /* *------------------------------------------------------------------------------ * Function declarations and virtual trampolines @@ -138,42 +112,25 @@ rc_allocator_recovery_record_reference_virtual(allocator *a, } platform_status -rc_allocator_get_super_addr(rc_allocator *al, - allocator_root_id spl_id, - uint64 *addr); +rc_allocator_open_refcounts(rc_allocator *al, bool32 rebuild); platform_status -rc_allocator_get_super_addr_virtual(allocator *a, - allocator_root_id spl_id, - uint64 *addr) +rc_allocator_open_refcounts_virtual(allocator *a, bool32 rebuild) { rc_allocator *al = (rc_allocator *)a; - return rc_allocator_get_super_addr(al, spl_id, addr); + return rc_allocator_open_refcounts(al, rebuild); } platform_status -rc_allocator_alloc_super_addr(rc_allocator *al, - allocator_root_id spl_id, - uint64 *addr); +rc_allocator_persist(rc_allocator *al, uint64 *state_addr); platform_status -rc_allocator_alloc_super_addr_virtual(allocator *a, - allocator_root_id spl_id, - uint64 *addr) +rc_allocator_persist_virtual(allocator *a, uint64 *state_addr) { rc_allocator *al = (rc_allocator *)a; - return rc_allocator_alloc_super_addr(al, spl_id, addr); + return rc_allocator_persist(al, state_addr); } -void -rc_allocator_remove_super_addr(rc_allocator *al, allocator_root_id spl_id); - -void -rc_allocator_remove_super_addr_virtual(allocator *a, allocator_root_id spl_id) -{ - rc_allocator *al = (rc_allocator *)a; - rc_allocator_remove_super_addr(al, spl_id); -} uint64 rc_allocator_in_use(rc_allocator *al); @@ -232,9 +189,8 @@ const static allocator_ops rc_allocator_ops = { .dec_ref = rc_allocator_dec_ref_virtual, .get_ref = rc_allocator_get_ref_virtual, .recovery_record_reference = rc_allocator_recovery_record_reference_virtual, - .get_super_addr = rc_allocator_get_super_addr_virtual, - .alloc_super_addr = rc_allocator_alloc_super_addr_virtual, - .remove_super_addr = rc_allocator_remove_super_addr_virtual, + .open_refcounts = rc_allocator_open_refcounts_virtual, + .persist = rc_allocator_persist_virtual, .in_use = rc_allocator_in_use_virtual, .get_capacity = rc_allocator_get_capacity_virtual, .assert_noleaks = rc_allocator_assert_noleaks_virtual, @@ -268,34 +224,6 @@ rc_allocator_extent_number(rc_allocator *al, uint64 addr) return (addr / al->cfg->io_cfg->extent_size); } -static checksum128 -rc_allocator_meta_page_checksum(const rc_allocator_meta_page *meta_page) -{ - return platform_checksum128(meta_page, - offsetof(rc_allocator_meta_page, checksum), - RC_ALLOCATOR_META_PAGE_CSUM_SEED); -} - -static platform_status -rc_allocator_write_meta_page(rc_allocator *al) -{ - al->meta_page->checksum = rc_allocator_meta_page_checksum(al->meta_page); - return io_write(al->io, - al->meta_page, - al->cfg->io_cfg->page_size, - RC_ALLOCATOR_BASE_OFFSET); -} - -static disk_geometry -rc_allocator_config_get_disk_geometry(allocator_config *cfg) -{ - return (disk_geometry){ - .disk_size = cfg->capacity, - .page_size = cfg->io_cfg->page_size, - .extent_size = cfg->io_cfg->extent_size, - }; -} - static uint64 rc_allocator_refcount_buffer_size(const allocator_config *cfg) { @@ -311,189 +239,12 @@ rc_allocator_refcount_extent_count(const allocator_config *cfg) / cfg->io_cfg->extent_size; } -static uint64 -rc_allocator_clean_state_extent_no(const allocator_config *cfg, uint64 slot) -{ - platform_assert(slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS); - return 1 + rc_allocator_refcount_extent_count(cfg) + slot; -} - -static uint64 -rc_allocator_clean_state_addr(const allocator_config *cfg, uint64 slot) -{ - return rc_allocator_clean_state_extent_no(cfg, slot) - * cfg->io_cfg->extent_size; -} - static uint64 rc_allocator_reserved_extent_count(const allocator_config *cfg) { - return 1 + rc_allocator_refcount_extent_count(cfg) - + RC_ALLOCATOR_CLEAN_STATE_SLOTS; -} - -static checksum128 -rc_allocator_clean_state_checksum(const rc_allocator_clean_state *state) -{ - return platform_checksum128(state, - offsetof(rc_allocator_clean_state, checksum), - RC_ALLOCATOR_CLEAN_STATE_CSUM_SEED); -} - -static bool32 -rc_allocator_clean_state_is_valid(const rc_allocator_clean_state *state) -{ - return state->magic == RC_ALLOCATOR_CLEAN_STATE_MAGIC - && state->format_version == RC_ALLOCATOR_CLEAN_STATE_VERSION - && state->sequence != 0 - && (state->clean_shutdown == FALSE || state->clean_shutdown == TRUE) - && platform_checksum_is_equal( - state->checksum, rc_allocator_clean_state_checksum(state)); -} - -static platform_status -rc_allocator_read_clean_state(rc_allocator *al, - uint64 slot, - rc_allocator_clean_state *state, - bool32 *valid) -{ - platform_assert(slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS); - platform_assert(sizeof(*state) <= al->cfg->io_cfg->page_size); - - buffer_handle buffer; - platform_status rc = - platform_buffer_init(&buffer, al->cfg->io_cfg->page_size); - if (!SUCCESS(rc)) { - return rc; - } - - void *page = platform_buffer_getaddr(&buffer); - rc = io_read(al->io, - page, - al->cfg->io_cfg->page_size, - rc_allocator_clean_state_addr(al->cfg, slot)); - if (SUCCESS(rc)) { - memcpy(state, page, sizeof(*state)); - *valid = rc_allocator_clean_state_is_valid(state); - } - - platform_status deinit_rc = platform_buffer_deinit(&buffer); - if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { - rc = deinit_rc; - } - return rc; -} - -static platform_status -rc_allocator_write_clean_state(rc_allocator *al, - uint64 slot, - const rc_allocator_clean_state *state, - bool32 durable) -{ - platform_assert(slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS); - platform_assert(sizeof(*state) <= al->cfg->io_cfg->page_size); - - buffer_handle buffer; - platform_status rc = - platform_buffer_init(&buffer, al->cfg->io_cfg->page_size); - if (!SUCCESS(rc)) { - return rc; - } - - void *page = platform_buffer_getaddr(&buffer); - memset(page, 0, al->cfg->io_cfg->page_size); - memcpy(page, state, sizeof(*state)); - rc = io_write(al->io, - page, - al->cfg->io_cfg->page_size, - rc_allocator_clean_state_addr(al->cfg, slot)); - if (SUCCESS(rc) && durable) { - rc = io_durable_barrier(al->io); - } - - platform_status deinit_rc = platform_buffer_deinit(&buffer); - if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { - rc = deinit_rc; - } - return rc; -} - -static platform_status -rc_allocator_load_clean_states(rc_allocator *al, - rc_allocator_clean_states *states) -{ - ZERO_CONTENTS(states); - for (uint64 slot = 0; slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS; slot++) { - platform_status rc = rc_allocator_read_clean_state( - al, slot, &states->state[slot], &states->valid[slot]); - if (!SUCCESS(rc)) { - return rc; - } - - if (!states->valid[slot]) { - continue; - } - if (!states->have_newest - || states->state[states->newest_slot].sequence - < states->state[slot].sequence) - { - states->have_newest = TRUE; - states->newest_slot = slot; - } else if (states->state[states->newest_slot].sequence - == states->state[slot].sequence) - { - states->duplicate_sequence = TRUE; - } - } - return STATUS_OK; -} - -static platform_status -rc_allocator_publish_clean_state(rc_allocator *al, bool32 clean_shutdown) -{ - rc_allocator_clean_states states; - platform_status rc = rc_allocator_load_clean_states(al, &states); - if (!SUCCESS(rc)) { - return rc; - } - - if (states.have_newest - && states.state[states.newest_slot].sequence == UINT64_MAX) - { - return STATUS_LIMIT_EXCEEDED; - } - - uint64 target_slot = states.have_newest ? states.newest_slot ^ 1 : 0; - rc_allocator_clean_state state; - ZERO_CONTENTS(&state); - state.magic = RC_ALLOCATOR_CLEAN_STATE_MAGIC; - state.format_version = RC_ALLOCATOR_CLEAN_STATE_VERSION; - state.sequence = - states.have_newest ? states.state[states.newest_slot].sequence + 1 : 1; - state.clean_shutdown = clean_shutdown; - state.checksum = rc_allocator_clean_state_checksum(&state); - return rc_allocator_write_clean_state(al, target_slot, &state, TRUE); -} - -static platform_status -rc_allocator_initialize_clean_states(rc_allocator *al) -{ - for (uint64 slot = 0; slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS; slot++) { - rc_allocator_clean_state state; - ZERO_CONTENTS(&state); - state.magic = RC_ALLOCATOR_CLEAN_STATE_MAGIC; - state.format_version = RC_ALLOCATOR_CLEAN_STATE_VERSION; - state.sequence = slot + 1; - state.clean_shutdown = FALSE; - state.checksum = rc_allocator_clean_state_checksum(&state); - - platform_status rc = - rc_allocator_write_clean_state(al, slot, &state, FALSE); - if (!SUCCESS(rc)) { - return rc; - } - } - return io_durable_barrier(al->io); + /* Extent 0 (superblock) + the refcount-map extents. No clean-state + * extents: superblock allocation_state_addr now records map validity. */ + return 1 + rc_allocator_refcount_extent_count(cfg); } static void @@ -524,9 +275,8 @@ rc_allocator_recovery_initialize_refcounts(rc_allocator *al) memset(al->ref_count, 0, rc_allocator_refcount_buffer_size(al->cfg)); /* - * Extent 0 contains both the allocator meta page and every fixed table - * superblock. The refcount table begins at extent 1; the two extents - * after it hold alternating clean-state records. + * Extent 0 holds the superblock; the refcount map begins at extent 1. + * Reserve both so recovery never hands them out. */ for (uint64 extent_no = 0; extent_no < reserved_extent_count; extent_no++) { platform_assert(al->ref_count[extent_no] == AL_FREE); @@ -546,81 +296,6 @@ rc_allocator_recovery_extent_is_reserved(const rc_allocator *al, return extent_no < rc_allocator_reserved_extent_count(al->cfg); } -static platform_status -rc_allocator_validate_disk_geometry(rc_allocator *al) -{ - disk_geometry geometry = al->meta_page->geometry; - - return rc_allocator_disk_geometry_matches_config(&geometry, al->cfg); -} - -platform_status -rc_allocator_disk_geometry_matches_config(const disk_geometry *geometry, - const allocator_config *cfg) -{ - if (geometry->disk_size != cfg->capacity - || geometry->page_size != cfg->io_cfg->page_size - || geometry->extent_size != cfg->io_cfg->extent_size) - { - platform_error_log( - "SplinterDB disk geometry does not match configuration: " - "disk=(disk_size=%lu, page_size=%lu, extent_size=%lu), " - "config=(disk_size=%lu, page_size=%lu, extent_size=%lu)\n", - geometry->disk_size, - geometry->page_size, - geometry->extent_size, - cfg->capacity, - cfg->io_cfg->page_size, - cfg->io_cfg->extent_size); - return STATUS_BAD_PARAM; - } - - return STATUS_OK; -} - -platform_status -rc_allocator_read_disk_geometry(const char *filename, disk_geometry *geometry) -{ - return io_read_bootstrap( - filename, geometry, sizeof(*geometry), RC_ALLOCATOR_BASE_OFFSET); -} - -static platform_status -rc_allocator_init_meta_page(rc_allocator *al) -{ - /* - * To make it easier to do aligned i/o's we allocate the meta page to - * always be page size. In the future we can use the remaining space - * for some other reserved information we may want to persist as part - * of the meta page. - */ - platform_assert(sizeof(rc_allocator_meta_page) - <= al->cfg->io_cfg->page_size); - /* - * Ensure that the meta page and all the super blocks will fit in one - * extent. - */ - platform_assert((1 + RC_ALLOCATOR_MAX_ROOT_IDS) * al->cfg->io_cfg->page_size - <= al->cfg->io_cfg->extent_size); - - al->meta_page = TYPED_ALIGNED_ZALLOC(al->heap_id, - al->cfg->io_cfg->page_size, - al->meta_page, - al->cfg->io_cfg->page_size); - if (al->meta_page == NULL) { - return STATUS_NO_MEMORY; - } - - memset(al->meta_page->splinters, - INVALID_ALLOCATOR_ROOT_ID, - sizeof(al->meta_page->splinters)); - al->meta_page->geometry = rc_allocator_config_get_disk_geometry(al->cfg); - al->meta_page->format_magic = RC_ALLOCATOR_FORMAT_MAGIC; - al->meta_page->format_version = RC_ALLOCATOR_FORMAT_VERSION; - - return STATUS_OK; -} - /* *---------------------------------------------------------------------- * rc_allocator_valid_config() -- @@ -719,69 +394,36 @@ rc_allocator_init(rc_allocator *al, platform_error_log("Failed to init mutex for the allocator\n"); return rc; } - rc = rc_allocator_init_meta_page(al); - if (!SUCCESS(rc)) { - platform_error_log("Failed to init meta page for rc allocator\n"); - platform_mutex_destroy(&al->lock); - return rc; - } // To ensure alignment always allocate in multiples of page size. uint64 buffer_size = rc_allocator_refcount_buffer_size(cfg); rc = platform_buffer_init(&al->bh, buffer_size); if (!SUCCESS(rc)) { platform_mutex_destroy(&al->lock); - platform_free(al->heap_id, al->meta_page); platform_error_log("Failed to create buffer for ref counts\n"); return STATUS_NO_MEMORY; } al->ref_count = platform_buffer_getaddr(&al->bh); memset(al->ref_count, 0, buffer_size); - // allocate the super block + // Reserve extent 0 for the superblock; the superblock module owns its + // pages 0/1. allocator_alloc(&al->super, &addr, PAGE_TYPE_SUPERBLOCK); - // super block extent should always start from address 0. - platform_assert(addr == RC_ALLOCATOR_BASE_OFFSET); + platform_assert(addr == 0); - /* - * Allocate room for the ref counts, use same rounded up size used in buffer - * creation. - */ + // Reserve the refcount-map extents (extent 1 .. rc_extent_count). rc_extent_count = rc_allocator_refcount_extent_count(cfg); for (uint64 i = 0; i < rc_extent_count; i++) { allocator_alloc(&al->super, &addr, PAGE_TYPE_SUPERBLOCK); platform_assert(addr == cfg->io_cfg->extent_size * (i + 1)); } - for (uint64 slot = 0; slot < RC_ALLOCATOR_CLEAN_STATE_SLOTS; slot++) { - allocator_alloc(&al->super, &addr, PAGE_TYPE_SUPERBLOCK); - platform_assert(addr == rc_allocator_clean_state_addr(cfg, slot)); - } - /* - * Persist the immutable bootstrap layout and both initial false state - * records before returning a newly created allocator. A crash before a - * later clean close therefore enters rebuild recovery rather than trusting - * this freshly initialized refcount map. + * A fresh allocator's refcount map is not persisted here. The superblock + * that mkfs writes records allocation_state_addr == 0, so a crash before a + * later clean close enters rebuild recovery rather than trusting this + * in-memory map. The map is persisted only by rc_allocator_persist(). */ - rc = rc_allocator_initialize_clean_states(al); - if (!SUCCESS(rc)) { - goto deinit_allocator; - } - rc = rc_allocator_write_meta_page(al); - if (!SUCCESS(rc)) { - goto deinit_allocator; - } - rc = io_durable_barrier(al->io); - if (!SUCCESS(rc)) { - goto deinit_allocator; - } - return STATUS_OK; - -deinit_allocator: - rc_allocator_deinit(al); - ZERO_CONTENTS(al); - return rc; } void @@ -790,24 +432,25 @@ rc_allocator_deinit(rc_allocator *al) platform_buffer_deinit(&al->bh); al->ref_count = NULL; platform_mutex_destroy(&al->lock); - platform_free(al->heap_id, al->meta_page); } /* *---------------------------------------------------------------------- - * rc_allocator_{mount,unmount} -- + * rc_allocator_mount -- * - * Loads the file system from disk - * Write the file system to disk + * Attach an allocator to an existing device: initialize the in-memory + * structures and allocate the (zeroed) refcount buffer, but do NOT read + * the persisted map. The caller populates the map exactly once via + * allocator_open_refcounts() -- loading the trusted map or initializing a + * rebuild -- so a rebuild never pays for a map read it would discard. *---------------------------------------------------------------------- */ -static platform_status -rc_allocator_mount_internal(rc_allocator *al, - allocator_config *cfg, - io_handle *io, - platform_heap_id hid, - platform_module_id mid, - bool32 rebuild_refcounts) +platform_status +rc_allocator_mount(rc_allocator *al, + allocator_config *cfg, + io_handle *io, + platform_heap_id hid, + platform_module_id mid) { platform_status status; @@ -824,13 +467,6 @@ rc_allocator_mount_internal(rc_allocator *al, return status; } - status = rc_allocator_init_meta_page(al); - if (!SUCCESS(status)) { - platform_error_log("Failed to init meta page for rc allocator\n"); - platform_mutex_destroy(&al->lock); - return status; - } - platform_assert(cfg->io_cfg->page_size % 4096 == 0); platform_assert(cfg->capacity == cfg->io_cfg->extent_size * cfg->extent_capacity); @@ -840,118 +476,64 @@ rc_allocator_mount_internal(rc_allocator *al, uint64 buffer_size = rc_allocator_refcount_buffer_size(cfg); status = platform_buffer_init(&al->bh, buffer_size); if (!SUCCESS(status)) { - platform_free(al->heap_id, al->meta_page); platform_mutex_destroy(&al->lock); platform_error_log("Failed to create buffer to load ref counts\n"); return STATUS_NO_MEMORY; } al->ref_count = platform_buffer_getaddr(&al->bh); + memset(al->ref_count, 0, buffer_size); + return STATUS_OK; +} - // load the meta page from disk. - status = io_read( - io, al->meta_page, al->cfg->io_cfg->page_size, RC_ALLOCATOR_BASE_OFFSET); - if (!SUCCESS(status)) { - goto deinit_buffer; - } - - status = rc_allocator_validate_disk_geometry(al); - if (!SUCCESS(status)) { - goto deinit_buffer; - } - - // validate the checksum of the meta page. - checksum128 currChecksum = rc_allocator_meta_page_checksum(al->meta_page); - if (!platform_checksum_is_equal(al->meta_page->checksum, currChecksum)) { - platform_error_log("Corrupt SplinterDB allocator meta page on mount\n"); - status = STATUS_BAD_PARAM; - goto deinit_buffer; - } - - if (al->meta_page->format_magic != RC_ALLOCATOR_FORMAT_MAGIC - || al->meta_page->format_version != RC_ALLOCATOR_FORMAT_VERSION) - { - platform_error_log("Unsupported SplinterDB allocator bootstrap format " - "on mount.\n"); - status = STATUS_BAD_PARAM; - goto deinit_buffer; - } - - if (!rebuild_refcounts) { - rc_allocator_clean_states states; - status = rc_allocator_load_clean_states(al, &states); - if (!SUCCESS(status)) { - goto deinit_buffer; - } - if (!states.have_newest || states.duplicate_sequence - || !states.state[states.newest_slot].clean_shutdown) - { - platform_error_log("Allocator was not cleanly shut down; recovery " - "rebuild is required.\n"); - status = STATUS_INVALID_STATE; - goto deinit_buffer; - } - } +/* + *---------------------------------------------------------------------- + * rc_allocator_open_refcounts -- + * + * Populate the refcount map of an attached allocator (see + * allocator_open_refcounts()). rebuild == FALSE loads the trusted + * persisted map from its fixed reserved location; rebuild == TRUE + * initializes an empty map that reserves only the fixed extents and enters + * recovery mode. Geometry and clean-vs-rebuild validity live in the + * superblock, which the caller has already read; the allocator only + * touches its own refcount map here. + *---------------------------------------------------------------------- + */ +platform_status +rc_allocator_open_refcounts(rc_allocator *al, bool32 rebuild) +{ + platform_assert(al != NULL); + platform_assert(al->ref_count != NULL); - if (rebuild_refcounts) { - /* Do not leave a stale clean state if recovery itself is interrupted. */ - status = rc_allocator_publish_clean_state(al, FALSE); - if (!SUCCESS(status)) { - goto deinit_buffer; - } - status = rc_allocator_recovery_initialize_refcounts(al); + if (rebuild) { + platform_status status = rc_allocator_recovery_initialize_refcounts(al); if (!SUCCESS(status)) { - goto deinit_buffer; + return status; } al->recovery_in_progress = TRUE; - } else { - // Load the ref counts from disk during a normal, clean mount. - status = - io_read(io, al->ref_count, buffer_size, cfg->io_cfg->extent_size); - if (!SUCCESS(status)) { - goto deinit_buffer; - } + return STATUS_OK; + } - for (uint64 i = 0; i < al->cfg->extent_capacity; i++) { - if (al->ref_count[i] != 0) { - al->stats.curr_allocated++; - } - } + // Load the trusted refcount map from its fixed reserved location. + uint64 buffer_size = rc_allocator_refcount_buffer_size(al->cfg); + platform_status status = io_read(al->io, + al->ref_count, + buffer_size, + RC_ALLOCATOR_REFCOUNT_MAP_EXTENT + * al->cfg->io_cfg->extent_size); + if (!SUCCESS(status)) { + return status; + } - /* Mark dirty before handing the trusted map to any mutating caller. */ - status = rc_allocator_publish_clean_state(al, FALSE); - if (!SUCCESS(status)) { - goto deinit_buffer; + // Compute curr_allocated authoritatively from the loaded map (set, not + // accumulate), so this is correct even if the allocator's stats were not + // freshly zeroed. + al->stats.curr_allocated = 0; + for (uint64 i = 0; i < al->cfg->extent_capacity; i++) { + if (al->ref_count[i] != 0) { + al->stats.curr_allocated++; } } return STATUS_OK; - -deinit_buffer: - platform_buffer_deinit(&al->bh); - al->ref_count = NULL; - platform_free(al->heap_id, al->meta_page); - al->meta_page = NULL; - platform_mutex_destroy(&al->lock); - return status; -} - -platform_status -rc_allocator_mount(rc_allocator *al, - allocator_config *cfg, - io_handle *io, - platform_heap_id hid, - platform_module_id mid) -{ - return rc_allocator_mount_internal(al, cfg, io, hid, mid, FALSE); -} - -platform_status -rc_allocator_mount_recovery(rc_allocator *al, - allocator_config *cfg, - io_handle *io, - platform_heap_id hid, - platform_module_id mid) -{ - return rc_allocator_mount_internal(al, cfg, io, hid, mid, TRUE); } platform_status @@ -1031,38 +613,40 @@ rc_allocator_abort_recovery(rc_allocator *al) } -void -rc_allocator_unmount(rc_allocator *al) +/* + *---------------------------------------------------------------------- + * rc_allocator_persist -- + * + * Write the refcount map to its fixed reserved location and make it + * durable. Returns the base address of the persisted map in *state_addr, + * which the caller records as the superblock's allocation_state_addr only + * after this returns -- so the "map is trustworthy" flag never becomes + * durable before the map itself. Does not tear down the allocator; the + * caller invokes rc_allocator_deinit() separately. + *---------------------------------------------------------------------- + */ +platform_status +rc_allocator_persist(rc_allocator *al, uint64 *state_addr) { - platform_status status; + platform_assert(!al->recovery_in_progress); - if (al->recovery_in_progress) { - platform_error_log("Discarding incomplete allocator recovery instead of " - "persisting its partial refcount map.\n"); - rc_allocator_abort_recovery(al); - return; - } - - // persist the ref counts upon unmount. + uint64 map_addr = + RC_ALLOCATOR_REFCOUNT_MAP_EXTENT * al->cfg->io_cfg->extent_size; uint64 buffer_size = rc_allocator_refcount_buffer_size(al->cfg); uint32 io_size = ROUNDUP(buffer_size, al->cfg->io_cfg->page_size); - status = - io_write(al->io, al->ref_count, io_size, al->cfg->io_cfg->extent_size); - platform_assert_status_ok(status); - /* - * This is the sole normal persistence point for the allocator map. The - * checkpoint record was already made durable by core_unmount(); do not - * advertise a clean allocator snapshot until this write has crossed the - * device durability boundary as well. - */ + platform_status status = io_write(al->io, al->ref_count, io_size, map_addr); + if (!SUCCESS(status)) { + return status; + } status = io_durable_barrier(al->io); - platform_assert_status_ok(status); - - /* Publish clean permission in the alternate durable state record. */ - status = rc_allocator_publish_clean_state(al, TRUE); - platform_assert_status_ok(status); - rc_allocator_deinit(al); + if (!SUCCESS(status)) { + return status; + } + if (state_addr != NULL) { + *state_addr = map_addr; + } + return STATUS_OK; } @@ -1153,76 +737,6 @@ rc_allocator_get_capacity(rc_allocator *al) return al->cfg->capacity; } -platform_status -rc_allocator_get_super_addr(rc_allocator *al, - allocator_root_id allocator_root_id, - uint64 *addr) -{ - platform_status status = STATUS_NOT_FOUND; - - platform_mutex_lock(&al->lock); - for (uint8 idx = 0; idx < RC_ALLOCATOR_MAX_ROOT_IDS; idx++) { - if (al->meta_page->splinters[idx] == allocator_root_id) { - // have already seen this table before, return existing addr. - *addr = (1 + idx) * al->cfg->io_cfg->page_size; - status = STATUS_OK; - break; - } - } - - platform_mutex_unlock(&al->lock); - return status; -} - -platform_status -rc_allocator_alloc_super_addr(rc_allocator *al, - allocator_root_id allocator_root_id, - uint64 *addr) -{ - platform_status status = STATUS_NOT_FOUND; - - platform_mutex_lock(&al->lock); - for (uint8 idx = 0; idx < RC_ALLOCATOR_MAX_ROOT_IDS; idx++) { - if (al->meta_page->splinters[idx] == INVALID_ALLOCATOR_ROOT_ID) { - // assign the first available slot and update the on disk metadata. - al->meta_page->splinters[idx] = allocator_root_id; - *addr = (1 + idx) * al->cfg->io_cfg->page_size; - platform_status io_status = rc_allocator_write_meta_page(al); - platform_assert_status_ok(io_status); - status = STATUS_OK; - break; - } - } - - platform_mutex_unlock(&al->lock); - return status; -} - -void -rc_allocator_remove_super_addr(rc_allocator *al, - allocator_root_id allocator_root_id) -{ - platform_mutex_lock(&al->lock); - - for (uint8 idx = 0; idx < RC_ALLOCATOR_MAX_ROOT_IDS; idx++) { - /* - * clear out the mapping for this splinter table and update on disk - * metadata. - */ - if (al->meta_page->splinters[idx] == allocator_root_id) { - al->meta_page->splinters[idx] = INVALID_ALLOCATOR_ROOT_ID; - platform_status status = rc_allocator_write_meta_page(al); - platform_assert_status_ok(status); - platform_mutex_unlock(&al->lock); - return; - } - } - - platform_mutex_unlock(&al->lock); - // Couldn't find the splinter id in the meta page. - platform_assert(0, "Couldn't find existing splinter table in meta page"); -} - uint64 rc_allocator_extent_size(rc_allocator *al) { diff --git a/src/rc_allocator.h b/src/rc_allocator.h index b7ada5cc..31c0980b 100644 --- a/src/rc_allocator.h +++ b/src/rc_allocator.h @@ -15,41 +15,6 @@ #include "allocator.h" #include "util.h" -/* - * In the current system, every Splinter instance has a superblock, one - * for each table that is mapped to the Splinter instance. This limit - * is the max number of superblocks (special pages) that can be accessed. - * All of these superblocks are required to be on the 1st extent. - */ -#define RC_ALLOCATOR_MAX_ROOT_IDS (30) - -/* - *---------------------------------------------------------------------- - * rc_allocator_meta_page -- Disk-resident structure. - * - * An on disk structure to hold the bootstrap disk geometry and the super block - * addresses for all Splinter tables using this allocator. The geometry lives at - * offset 0 so open can read it before mounting the rest of SplinterDB. - *---------------------------------------------------------------------- - */ -typedef struct ONDISK rc_allocator_meta_page { - disk_geometry geometry; - /* - * Identifies the immutable bootstrap layout. In particular, it proves - * that the two fixed clean-state extents after the refcount map are owned - * by this allocator rather than by an older on-disk format. - */ - uint64 format_magic; - uint64 format_version; - allocator_root_id splinters[RC_ALLOCATOR_MAX_ROOT_IDS]; - checksum128 checksum; -} rc_allocator_meta_page; - -_Static_assert(offsetof(rc_allocator_meta_page, geometry) == 0, - "disk geometry should be first field in meta_page struct"); -_Static_assert(sizeof(rc_allocator_meta_page) <= IO_DEFAULT_PAGE_SIZE, - "allocator meta page must fit in the default page size"); - /* *---------------------------------------------------------------------- * rc_allocator_stats -- @@ -68,23 +33,19 @@ typedef struct rc_allocator_stats { *---------------------------------------------------------------------- */ typedef struct rc_allocator { - allocator super; - allocator_config *cfg; - buffer_handle bh; - refcount *ref_count; - uint64 hand; - io_handle *io; - rc_allocator_meta_page *meta_page; - - /* - * mutex to synchronize updates to super block addresses of the splinter - * tables in the meta page. - */ + allocator super; + allocator_config *cfg; + buffer_handle bh; + refcount *ref_count; + uint64 hand; + io_handle *io; + + /* Serializes refcount-map mutations that must be atomic (e.g. stats). */ platform_mutex lock; platform_heap_id heap_id; /* - * True between rc_allocator_mount_recovery() and either + * True between rc_allocator_open_refcounts(al, rebuild=TRUE) and either * rc_allocator_rebuild_finish() or rc_allocator_abort_recovery(). An * incomplete rebuilt map must never be written back to disk. */ @@ -105,12 +66,15 @@ void rc_allocator_deinit(rc_allocator *al); /* - * Normal mount accepts only a durable clean-state record and publishes an - * unclean state record before returning. Those records occupy two fixed, - * alternating extents after the persisted refcount map, so normal lifecycle - * operations never rewrite the sole allocator bootstrap page. - * STATUS_INVALID_STATE means callers must use the recovery-rebuild path - * instead of trusting the persisted refcount map. + * Attach to an existing device: initialize the in-memory structures and the + * (zeroed) refcount buffer, but do not read the persisted map. The caller + * then populates the map exactly once via allocator_open_refcounts() (see + * allocator.h): open_refcounts(rebuild=FALSE) loads the trusted persisted map; + * open_refcounts(rebuild=TRUE) initializes an empty map for a rebuild. The + * caller chooses the mode from the superblock's allocation-state validity, + * which it has already read, so a rebuild never pays for a discarded map read. + * Before any allocation mutates the loaded map, the caller publishes an + * invalidated superblock so a crash forces a rebuild. */ platform_status rc_allocator_mount(rc_allocator *al, @@ -119,20 +83,6 @@ rc_allocator_mount(rc_allocator *al, platform_heap_id hid, platform_module_id mid); -/* - * Mount the allocator for crash recovery. This validates and retains the - * allocator metadata page, but deliberately ignores the persisted refcount - * table. The caller must rebuild the in-memory table from durable objects, - * then call rc_allocator_rebuild_finish() before using normal allocator - * lifecycle operations. - */ -platform_status -rc_allocator_mount_recovery(rc_allocator *al, - allocator_config *cfg, - io_handle *io, - platform_heap_id hid, - platform_module_id mid); - /* * Add one logical ownership reference for an extent while rebuilding a * recovery map. The first reference establishes the allocator's nonzero @@ -146,25 +96,15 @@ rc_allocator_rebuild_acquire_extent(rc_allocator *al, page_type type); /* - * Complete a successful rebuild without performing I/O. A later normal - * rc_allocator_unmount() may persist the rebuilt table on clean shutdown. + * Complete a successful rebuild without performing I/O. A later + * rc_allocator_persist() may persist the rebuilt table on clean shutdown. */ void rc_allocator_rebuild_finish(rc_allocator *al); /* - * Discard a partially rebuilt recovery map. Unlike rc_allocator_unmount(), - * this never writes allocator state to disk. + * Discard a partially rebuilt recovery map. Never writes allocator state to + * disk; the caller follows with rc_allocator_deinit(). */ void rc_allocator_abort_recovery(rc_allocator *al); - -platform_status -rc_allocator_read_disk_geometry(const char *filename, disk_geometry *geometry); - -platform_status -rc_allocator_disk_geometry_matches_config(const disk_geometry *geometry, - const allocator_config *cfg); - -void -rc_allocator_unmount(rc_allocator *al); diff --git a/src/splinterdb.c b/src/splinterdb.c index c40dc8c3..35ce2384 100644 --- a/src/splinterdb.c +++ b/src/splinterdb.c @@ -17,6 +17,7 @@ #include "clockcache.h" #include "data_internal.h" #include "rc_allocator.h" +#include "superblock.h" #include "core.h" #include "lookup_result.h" #include "notification.h" @@ -243,10 +244,24 @@ splinterdb_init_config(const splinterdb_config *kvs_cfg, // IN allocator_config_init(&kvs->allocator_cfg, &kvs->io_cfg, cfg.disk_size); if (geometry != NULL) { - rc = rc_allocator_disk_geometry_matches_config(geometry, - &kvs->allocator_cfg); - if (!SUCCESS(rc)) { - return rc; + // The auto-read on-disk geometry must agree with the resolved config; + // core_mount re-validates it against the superblock, but check here for + // an early, clear error when the caller overrides a device field. + if (geometry->disk_size != kvs->allocator_cfg.capacity + || geometry->page_size != kvs->io_cfg.page_size + || geometry->extent_size != kvs->io_cfg.extent_size) + { + platform_error_log( + "On-disk geometry (disk_size=%lu page_size=%lu extent_size=%lu) " + "does not match the configured geometry (disk_size=%lu " + "page_size=%lu extent_size=%lu).\n", + geometry->disk_size, + geometry->page_size, + geometry->extent_size, + kvs->allocator_cfg.capacity, + kvs->io_cfg.page_size, + kvs->io_cfg.extent_size); + return STATUS_BAD_PARAM; } } @@ -315,7 +330,7 @@ splinterdb_config_read_disk_geometry(const char *filename, return STATUS_BAD_PARAM; } - platform_status status = rc_allocator_read_disk_geometry(filename, geometry); + platform_status status = superblock_read_geometry(filename, geometry); if (!SUCCESS(status)) { return status; } @@ -473,6 +488,7 @@ splinterdb_create_or_open(const splinterdb_config *kvs_cfg, // IN &kvs->trunk_cfg, (allocator *)&kvs->allocator_handle, (cache *)&kvs->cache_handle, + kvs->io_handle, &kvs->task_sys, kvs->trunk_id, kvs->heap_id); @@ -481,6 +497,7 @@ splinterdb_create_or_open(const splinterdb_config *kvs_cfg, // IN &kvs->trunk_cfg, (allocator *)&kvs->allocator_handle, (cache *)&kvs->cache_handle, + kvs->io_handle, &kvs->task_sys, kvs->trunk_id, kvs->heap_id); @@ -579,13 +596,13 @@ splinterdb_close(splinterdb **kvs_in) // IN } io_wait_all(kvs->io_handle); clockcache_deinit(&kvs->cache_handle); - if (SUCCESS(status)) { - /* This writes the map and only then publishes allocator clean state. */ - rc_allocator_unmount(&kvs->allocator_handle); - } else { - /* Keep the durable clean marker cleared; the next open must rebuild. */ - rc_allocator_deinit(&kvs->allocator_handle); - } + /* + * core_unmount() already persisted the refcount map and published the + * superblock's allocation state (or, on failure, deliberately left it + * invalid so the next open rebuilds); the allocator now only tears down its + * in-memory structures. + */ + rc_allocator_deinit(&kvs->allocator_handle); task_system_deinit(&kvs->task_sys); io_handle_destroy(kvs->io_handle); diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index 6dd55db3..c5f8a600 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -146,256 +146,208 @@ cache_test_verify_disk_page(cache *cc, } /* - * Corrupt one full raw-I/O page and make that corruption durable. The - * allocator clean-state records are intentionally outside the cache, so this - * lets the bootstrap test exercise A/B fallback without test-only allocator - * hooks. - */ -static platform_status -cache_test_zero_disk_page(io_handle *io, uint64 addr, uint64 page_size) -{ - buffer_handle buffer; - platform_status rc = platform_buffer_init(&buffer, page_size); - if (!SUCCESS(rc)) { - return rc; - } - - void *page = platform_buffer_getaddr(&buffer); - memset(page, 0, page_size); - rc = io_write(io, page, page_size, addr); - if (SUCCESS(rc)) { - rc = io_durable_barrier(io); - } - - platform_status deinit_rc = platform_buffer_deinit(&buffer); - if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { - rc = deinit_rc; - } - return rc; -} - -/* - * The recovery allocator must bootstrap only the extents it owns itself, then - * let recovery walkers rebuild all other ownership. In particular, it must - * not trust a refcount table persisted by an earlier clean shutdown. + * The refcount allocator is a pure store: mount() attaches without reading the + * persisted map, and the caller then chooses how to populate it via + * allocator_open_refcounts(). rebuild == FALSE loads the trusted persisted + * map; rebuild == TRUE ignores it and reserves only the allocator's own fixed + * extents, leaving all other ownership for a caller-driven rebuild. + * persist() writes the map durably; abort_recovery() never persists. This + * exercises that contract end to end (clean-vs-rebuild is the caller's + * decision -- the superblock's -- not the allocator's). */ static platform_status test_rc_allocator_recovery_bootstrap(allocator_config *cfg, io_handle *io, platform_heap_id hid) { - const allocator_root_id root_id = 1; - uint64 refcount_buffer_size = + uint64 refcount_buffer_size = ROUNDUP(cfg->extent_capacity * sizeof(refcount), cfg->io_cfg->page_size); uint64 refcount_extent_count = (refcount_buffer_size + cfg->io_cfg->extent_size - 1) / cfg->io_cfg->extent_size; - uint64 reserved_extent_count = 1 + refcount_extent_count + 2; - uint64 old_clean_state_addr = - (1 + refcount_extent_count + 1) * cfg->io_cfg->extent_size; - uint64 super_addr = 0; - uint64 stale_extent_addr = 0; - platform_status rc = STATUS_OK; - rc_allocator original, recovery, remounted; - bool32 original_live = FALSE; - bool32 recovery_live = FALSE; - bool32 remounted_live = FALSE; - - ZERO_CONTENTS(&original); - ZERO_CONTENTS(&recovery); - ZERO_CONTENTS(&remounted); + // Extent 0 (superblock) + the refcount-map extents. + uint64 reserved_extent_count = 1 + refcount_extent_count; + uint64 stale_extent_addr = 0; + platform_status rc = STATUS_OK; + rc_allocator al; + bool32 al_live = FALSE; + + ZERO_CONTENTS(&al); platform_default_log( "cache_test: allocator recovery bootstrap test started\n"); - rc = rc_allocator_init(&original, cfg, io, hid, platform_get_module_id()); + // 1) Format, allocate a data extent, persist the map, tear down. + rc = rc_allocator_init(&al, cfg, io, hid, platform_get_module_id()); if (!SUCCESS(rc)) { goto cleanup; } - original_live = TRUE; - - rc = - allocator_alloc_super_addr((allocator *)&original, root_id, &super_addr); + al_live = TRUE; + rc = allocator_alloc((allocator *)&al, &stale_extent_addr, PAGE_TYPE_MISC); if (!SUCCESS(rc)) { goto cleanup; } - rc = allocator_alloc( - (allocator *)&original, &stale_extent_addr, PAGE_TYPE_MISC); + rc = allocator_persist((allocator *)&al, NULL); if (!SUCCESS(rc)) { goto cleanup; } + rc_allocator_deinit(&al); + al_live = FALSE; - /* Persist a non-reserved extent that the recovery mount must ignore. */ - rc_allocator_unmount(&original); - original_live = FALSE; - - /* - * New allocator initialization writes false records with sequences 1 and - * 2. The first clean close writes the true record into slot 0 (sequence - * 3), making slot 1 the older false record. Destroying that older slot - * must not prevent a normal mount from trusting the surviving true slot. - */ - rc = cache_test_zero_disk_page( - io, old_clean_state_addr, cfg->io_cfg->page_size); + // 2) Clean open: attach + open_refcounts(rebuild=FALSE) loads the map. + rc = rc_allocator_mount(&al, cfg, io, hid, platform_get_module_id()); if (!SUCCESS(rc)) { goto cleanup; } - - rc = rc_allocator_mount(&remounted, cfg, io, hid, platform_get_module_id()); + al_live = TRUE; + rc = allocator_open_refcounts((allocator *)&al, FALSE); if (!SUCCESS(rc)) { - platform_error_log("cache_test: normal mount did not fall back to its " - "surviving clean-state record\n"); goto cleanup; } - remounted_live = TRUE; - if (allocator_get_refcount((allocator *)&remounted, stale_extent_addr) + if (allocator_get_refcount((allocator *)&al, stale_extent_addr) != AL_ONE_REF) { - platform_error_log("cache_test: normal mount did not load the clean " - "refcount map\n"); - rc = STATUS_TEST_FAILED; - goto cleanup; - } - - /* Simulate a crash after normal mount durably marked the allocator dirty. */ - rc_allocator_deinit(&remounted); - remounted_live = FALSE; - - rc = rc_allocator_mount(&remounted, cfg, io, hid, platform_get_module_id()); - if (rc.r != STATUS_INVALID_STATE.r) { - platform_error_log("cache_test: normal mount trusted an allocator after " - "a simulated mounted crash\n"); + platform_error_log( + "cache_test: clean open did not load the persisted refcount map\n"); rc = STATUS_TEST_FAILED; goto cleanup; } - rc = STATUS_OK; + rc_allocator_deinit(&al); + al_live = FALSE; - rc = rc_allocator_mount_recovery( - &recovery, cfg, io, hid, platform_get_module_id()); + // 3) Rebuild open: attach + open_refcounts(rebuild=TRUE) ignores the + // persisted map, reserving only the fixed extents. + rc = rc_allocator_mount(&al, cfg, io, hid, platform_get_module_id()); if (!SUCCESS(rc)) { goto cleanup; } - recovery_live = TRUE; - - uint64 mounted_super_addr = 0; - rc = allocator_get_super_addr( - (allocator *)&recovery, root_id, &mounted_super_addr); - if (!SUCCESS(rc) || mounted_super_addr != super_addr) { - platform_error_log("cache_test: recovery lost a persisted superblock " - "mapping\n"); - rc = STATUS_TEST_FAILED; + al_live = TRUE; + rc = allocator_open_refcounts((allocator *)&al, TRUE); + if (!SUCCESS(rc)) { goto cleanup; } - if (allocator_in_use((allocator *)&recovery) != reserved_extent_count) { - platform_error_log("cache_test: recovery expected %lu reserved extents, " + if (allocator_in_use((allocator *)&al) != reserved_extent_count) { + platform_error_log("cache_test: rebuild expected %lu reserved extents, " "found %lu\n", reserved_extent_count, - allocator_in_use((allocator *)&recovery)); + allocator_in_use((allocator *)&al)); rc = STATUS_TEST_FAILED; goto cleanup; } for (uint64 extent_no = 0; extent_no < reserved_extent_count; extent_no++) { uint64 extent_addr = extent_no * cfg->io_cfg->extent_size; - if (allocator_get_refcount((allocator *)&recovery, extent_addr) - != AL_ONE_REF) - { - platform_error_log("cache_test: recovery did not reserve extent %lu\n", - extent_no); + if (allocator_get_refcount((allocator *)&al, extent_addr) != AL_ONE_REF) { + platform_error_log( + "cache_test: rebuild did not reserve fixed extent %lu\n", + extent_no); rc = STATUS_TEST_FAILED; goto cleanup; } } - if (allocator_get_refcount((allocator *)&recovery, stale_extent_addr) - != AL_FREE) - { - platform_error_log("cache_test: recovery reused a persisted data " - "refcount\n"); + if (allocator_get_refcount((allocator *)&al, stale_extent_addr) != AL_FREE) { + platform_error_log( + "cache_test: rebuild trusted a persisted data refcount\n"); rc = STATUS_TEST_FAILED; goto cleanup; } + // rebuild_acquire establishes the first reference. rc = rc_allocator_rebuild_acquire_extent( - &recovery, stale_extent_addr, PAGE_TYPE_MISC); + &al, stale_extent_addr, PAGE_TYPE_MISC); if (!SUCCESS(rc) - || allocator_get_refcount((allocator *)&recovery, stale_extent_addr) + || allocator_get_refcount((allocator *)&al, stale_extent_addr) != AL_ONE_REF) { - platform_error_log("cache_test: first rebuilt extent reference was " - "incorrect\n"); + platform_error_log( + "cache_test: first rebuilt extent reference was incorrect\n"); rc = STATUS_TEST_FAILED; goto cleanup; } - /* - * Abort never persists a partial rebuild and leaves the allocator marked - * unclean, so a normal mount must refuse to trust the old refcount map. - */ - rc_allocator_abort_recovery(&recovery); - recovery_live = FALSE; + // 4) Abort never persists: a later clean open still sees the pre-rebuild + // map (the aborted rebuild left no trace on disk). + rc_allocator_abort_recovery(&al); + al_live = FALSE; - rc = rc_allocator_mount(&remounted, cfg, io, hid, platform_get_module_id()); - if (rc.r != STATUS_INVALID_STATE.r) { - platform_error_log("cache_test: normal mount trusted an unclean " - "allocator map\n"); + rc = rc_allocator_mount(&al, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + al_live = TRUE; + rc = allocator_open_refcounts((allocator *)&al, FALSE); + if (!SUCCESS(rc)) { + goto cleanup; + } + if (allocator_get_refcount((allocator *)&al, stale_extent_addr) + != AL_ONE_REF) + { + platform_error_log( + "cache_test: aborted rebuild leaked into the persisted map\n"); rc = STATUS_TEST_FAILED; goto cleanup; } - rc = STATUS_OK; + rc_allocator_deinit(&al); + al_live = FALSE; - rc = rc_allocator_mount_recovery( - &recovery, cfg, io, hid, platform_get_module_id()); + // 5) Rebuild + finish + persist round-trips the rebuilt refcount. + rc = rc_allocator_mount(&al, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + al_live = TRUE; + rc = allocator_open_refcounts((allocator *)&al, TRUE); if (!SUCCESS(rc)) { goto cleanup; } - recovery_live = TRUE; - rc = rc_allocator_rebuild_acquire_extent( - &recovery, stale_extent_addr, PAGE_TYPE_MISC); + &al, stale_extent_addr, PAGE_TYPE_MISC); if (!SUCCESS(rc)) { goto cleanup; } rc = rc_allocator_rebuild_acquire_extent( - &recovery, stale_extent_addr, PAGE_TYPE_MISC); + &al, stale_extent_addr, PAGE_TYPE_MISC); if (!SUCCESS(rc) - || allocator_get_refcount((allocator *)&recovery, stale_extent_addr) + || allocator_get_refcount((allocator *)&al, stale_extent_addr) != AL_ONE_REF + 1) { - platform_error_log("cache_test: rebuilt extent reference did not " - "increment\n"); + platform_error_log( + "cache_test: rebuilt extent reference did not increment\n"); rc = STATUS_TEST_FAILED; goto cleanup; } - rc_allocator_rebuild_finish(&recovery); - rc_allocator_unmount(&recovery); - recovery_live = FALSE; + rc_allocator_rebuild_finish(&al); + rc = allocator_persist((allocator *)&al, NULL); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc_allocator_deinit(&al); + al_live = FALSE; - rc = rc_allocator_mount(&remounted, cfg, io, hid, platform_get_module_id()); + rc = rc_allocator_mount(&al, cfg, io, hid, platform_get_module_id()); + if (!SUCCESS(rc)) { + goto cleanup; + } + al_live = TRUE; + rc = allocator_open_refcounts((allocator *)&al, FALSE); if (!SUCCESS(rc)) { goto cleanup; } - remounted_live = TRUE; - if (allocator_get_refcount((allocator *)&remounted, stale_extent_addr) + if (allocator_get_refcount((allocator *)&al, stale_extent_addr) != AL_ONE_REF + 1) { - platform_error_log("cache_test: finished recovery was not persisted by " - "clean unmount\n"); + platform_error_log( + "cache_test: finished recovery was not persisted\n"); rc = STATUS_TEST_FAILED; goto cleanup; } cleanup: - if (remounted_live) { - rc_allocator_deinit(&remounted); - } - if (recovery_live) { - if (recovery.recovery_in_progress) { - rc_allocator_abort_recovery(&recovery); + if (al_live) { + if (al.recovery_in_progress) { + rc_allocator_abort_recovery(&al); } else { - rc_allocator_deinit(&recovery); + rc_allocator_deinit(&al); } } - if (original_live) { - rc_allocator_deinit(&original); - } if (SUCCESS(rc)) { platform_default_log("cache_test: allocator recovery bootstrap test " @@ -476,17 +428,25 @@ test_mini_recover_allocations(allocator_config *allocator_cfg, cache_flush(original_ccp); clockcache_deinit(&original_cc); original_cc_live = FALSE; - rc_allocator_unmount(&original); + rc = allocator_persist((allocator *)&original, NULL); + if (!SUCCESS(rc)) { + goto cleanup; + } + rc_allocator_deinit(&original); original_al_live = FALSE; - rc = rc_allocator_mount_recovery( + rc = rc_allocator_mount( &recovery, allocator_cfg, io, hid, platform_get_module_id()); if (!SUCCESS(rc)) { goto cleanup; } recovery_al_live = TRUE; + rc = allocator_open_refcounts((allocator *)&recovery, TRUE /* rebuild */); + if (!SUCCESS(rc)) { + goto cleanup; + } - /* mount_recovery deliberately ignores the persisted map: confirm that. */ + /* open_refcounts(rebuild) deliberately ignores the persisted map. */ if (allocator_get_refcount((allocator *)&recovery, meta_head) != AL_FREE) { platform_error_log( "cache_test: recovery mount trusted a persisted refcount\n"); diff --git a/tests/functional/splinter_test.c b/tests/functional/splinter_test.c index 5e665647..b8e641e2 100644 --- a/tests/functional/splinter_test.c +++ b/tests/functional/splinter_test.c @@ -789,6 +789,21 @@ test_trunk_create_tables(core_handle **spl_handles, uint8 num_tables, uint8 num_caches) { + /* + * The unified superblock is per-device: core owns one superblock per + * device, so multiple tables sharing one device/allocator is not supported + * until the multi-tree machinery lands (the on-disk format already reserves + * room for it). See the "core owns; 1 table/device" design decision. + */ + platform_assert(num_tables == 1, + "This test currently supports a single table per device " + "(num_tables=%u); rerun with --num-tables 1.", + num_tables); + + // These functional tests use rc_allocator exclusively; core_mkfs needs the + // io handle, which the allocator holds. + io_handle *io = ((rc_allocator *)al)->io; + core_handle *spl_tables = TYPED_ARRAY_ZALLOC(hid, spl_tables, num_tables); if (spl_tables == NULL) { return STATUS_NO_MEMORY; @@ -800,6 +815,7 @@ test_trunk_create_tables(core_handle **spl_handles, &cfg[spl_idx].splinter_cfg, al, cache_to_use, + io, ts, test_generate_allocator_root_id(), hid); diff --git a/tests/functional/test_functionality.c b/tests/functional/test_functionality.c index 7cac864b..d591e6e7 100644 --- a/tests/functional/test_functionality.c +++ b/tests/functional/test_functionality.c @@ -655,6 +655,17 @@ test_functionality(allocator *al, platform_error_log("Functional test started with %d tables\n", num_tables); platform_assert(cc != NULL); + /* + * The unified superblock is per-device: core owns one superblock per + * device, so multiple tables sharing one device/allocator is not supported + * until the multi-tree machinery lands (the on-disk format already reserves + * room for it). See the "core owns; 1 table/device" design decision. + */ + platform_assert(num_tables == 1, + "This test currently supports a single table per device " + "(num_tables=%u); rerun with --num-tables 1.", + num_tables); + core_handle *spl_tables = TYPED_ARRAY_ZALLOC(hid, spl_tables, num_tables); platform_assert(spl_tables != NULL); @@ -694,6 +705,7 @@ test_functionality(allocator *al, &cfg[idx].splinter_cfg, al, cache_to_use, + io, state, splinters[idx], hid); diff --git a/tests/functional/ycsb_test.c b/tests/functional/ycsb_test.c index 244f8300..7256f8ac 100644 --- a/tests/functional/ycsb_test.c +++ b/tests/functional/ycsb_test.c @@ -1295,6 +1295,7 @@ ycsb_test(int argc, char *argv[]) &system_cfg->splinter_cfg, (allocator *)&al, (cache *)cc, + io, &ts, test_generate_allocator_root_id(), hid); @@ -1314,6 +1315,7 @@ ycsb_test(int argc, char *argv[]) &system_cfg->splinter_cfg, (allocator *)&al, (cache *)cc, + io, &ts, test_generate_allocator_root_id(), hid); @@ -1325,7 +1327,8 @@ ycsb_test(int argc, char *argv[]) core_unmount(&spl); clockcache_deinit(cc); platform_free(hid, cc); - rc_allocator_unmount(&al); + // core_unmount() already persisted the map and published the superblock. + rc_allocator_deinit(&al); test_deinit_task_system(&ts); rc = STATUS_OK; diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index 01fe15f6..e7f9e2ee 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -235,6 +235,7 @@ CTEST2(splinter, test_inserts) &data->system_cfg->splinter_cfg, alp, (cache *)data->clock_cache, + data->io, &data->tasks, test_generate_allocator_root_id(), data->hid); @@ -268,6 +269,7 @@ CTEST2(splinter, test_mount_rejects_newer_active_checkpoint) &data->system_cfg->splinter_cfg, alp, (cache *)data->clock_cache, + data->io, &data->tasks, root_id, data->hid); @@ -294,6 +296,7 @@ CTEST2(splinter, test_mount_rejects_newer_active_checkpoint) &data->system_cfg->splinter_cfg, alp, (cache *)data->clock_cache, + data->io, &data->tasks, root_id, data->hid); @@ -303,6 +306,7 @@ CTEST2(splinter, test_mount_rejects_newer_active_checkpoint) &data->system_cfg->splinter_cfg, alp, (cache *)data->clock_cache, + data->io, &data->tasks, root_id, data->hid); @@ -316,6 +320,7 @@ CTEST2(splinter, test_mount_rejects_newer_active_checkpoint) &data->system_cfg->splinter_cfg, alp, (cache *)data->clock_cache, + data->io, &data->tasks, root_id, data->hid); @@ -482,6 +487,7 @@ CTEST2(splinter, test_lookups) &data->system_cfg->splinter_cfg, alp, (cache *)data->clock_cache, + data->io, &data->tasks, test_generate_allocator_root_id(), data->hid); @@ -707,6 +713,7 @@ CTEST2(splinter, test_splinter_print_diags) &data->system_cfg->splinter_cfg, alp, (cache *)data->clock_cache, + data->io, &data->tasks, test_generate_allocator_root_id(), data->hid); From eb052f562da95c8398f399a6b5ec7344b3192ec8 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 24 Jul 2026 11:27:06 -0700 Subject: [PATCH 15/64] rc_allocator cleanup Signed-off-by: Rob Johnson --- src/rc_allocator.c | 38 ++--- src/rc_allocator.h | 20 ++- src/superblock.c | 283 ++++++++++++++++++++++++++++++++++ src/superblock.h | 197 +++++++++++++++++++++++ tests/functional/cache_test.c | 21 +-- tests/unit/superblock_test.c | 237 ++++++++++++++++++++++++++++ 6 files changed, 749 insertions(+), 47 deletions(-) create mode 100644 src/superblock.c create mode 100644 src/superblock.h create mode 100644 tests/unit/superblock_test.c diff --git a/src/rc_allocator.c b/src/rc_allocator.c index dfb62236..3e120e78 100644 --- a/src/rc_allocator.c +++ b/src/rc_allocator.c @@ -418,11 +418,14 @@ rc_allocator_init(rc_allocator *al, } /* - * A fresh allocator's refcount map is not persisted here. The superblock - * that mkfs writes records allocation_state_addr == 0, so a crash before a - * later clean close enters rebuild recovery rather than trusting this - * in-memory map. The map is persisted only by rc_allocator_persist(). + * A fresh map is synthesized correctly in memory -- there is nothing to + * load, so it's trustworthy immediately, like a map that just finished a + * rebuild. It is not persisted here, though: the superblock that mkfs + * writes records allocation_state_addr == 0, so a crash before a later + * clean close enters rebuild recovery rather than trusting this in-memory + * map. The map is persisted only by rc_allocator_persist(). */ + al->map_is_valid = TRUE; return STATUS_OK; } @@ -509,7 +512,6 @@ rc_allocator_open_refcounts(rc_allocator *al, bool32 rebuild) if (!SUCCESS(status)) { return status; } - al->recovery_in_progress = TRUE; return STATUS_OK; } @@ -533,6 +535,7 @@ rc_allocator_open_refcounts(rc_allocator *al, bool32 rebuild) al->stats.curr_allocated++; } } + al->map_is_valid = TRUE; return STATUS_OK; } @@ -541,9 +544,9 @@ rc_allocator_rebuild_acquire_extent(rc_allocator *al, uint64 extent_addr, page_type type) { - if (!al->recovery_in_progress) { - platform_error_log("Cannot acquire allocator extent while recovery is " - "not in progress.\n"); + if (al->map_is_valid) { + platform_error_log( + "Cannot acquire allocator extent once the refcount map is valid.\n"); return STATUS_INVALID_STATE; } @@ -593,23 +596,14 @@ void rc_allocator_rebuild_finish(rc_allocator *al) { platform_assert(al != NULL); - platform_assert(al->recovery_in_progress); + platform_assert(!al->map_is_valid); /* * Intentionally no I/O here. A rebuilt map is durable only after a later - * clean unmount; another crash before then simply rebuilds it again. + * rc_allocator_persist(); another crash before then simply rebuilds it + * again. */ - al->recovery_in_progress = FALSE; -} - -void -rc_allocator_abort_recovery(rc_allocator *al) -{ - platform_assert(al != NULL); - platform_assert(al->recovery_in_progress); - - rc_allocator_deinit(al); - ZERO_CONTENTS(al); + al->map_is_valid = TRUE; } @@ -628,7 +622,7 @@ rc_allocator_abort_recovery(rc_allocator *al) platform_status rc_allocator_persist(rc_allocator *al, uint64 *state_addr) { - platform_assert(!al->recovery_in_progress); + platform_assert(al->map_is_valid); uint64 map_addr = RC_ALLOCATOR_REFCOUNT_MAP_EXTENT * al->cfg->io_cfg->extent_size; diff --git a/src/rc_allocator.h b/src/rc_allocator.h index 31c0980b..46996360 100644 --- a/src/rc_allocator.h +++ b/src/rc_allocator.h @@ -45,11 +45,14 @@ typedef struct rc_allocator { platform_heap_id heap_id; /* - * True between rc_allocator_open_refcounts(al, rebuild=TRUE) and either - * rc_allocator_rebuild_finish() or rc_allocator_abort_recovery(). An + * True once the refcount map is trustworthy: set by a clean + * rc_allocator_open_refcounts(al, rebuild=FALSE) load, or by + * rc_allocator_rebuild_finish() after a rebuild completes. False from + * rc_allocator_mount() (attach) until then, including throughout an + * in-progress rebuild. rc_allocator_persist() asserts this is true: an * incomplete rebuilt map must never be written back to disk. */ - bool32 recovery_in_progress; + bool32 map_is_valid; // Stats -- not distributed for now rc_allocator_stats stats; @@ -97,14 +100,9 @@ rc_allocator_rebuild_acquire_extent(rc_allocator *al, /* * Complete a successful rebuild without performing I/O. A later - * rc_allocator_persist() may persist the rebuilt table on clean shutdown. + * rc_allocator_persist() may persist the rebuilt table on clean shutdown. If + * the rebuild fails instead, there is nothing to undo: just + * rc_allocator_deinit() the allocator, since nothing has been persisted. */ void rc_allocator_rebuild_finish(rc_allocator *al); - -/* - * Discard a partially rebuilt recovery map. Never writes allocator state to - * disk; the caller follows with rc_allocator_deinit(). - */ -void -rc_allocator_abort_recovery(rc_allocator *al); diff --git a/src/superblock.c b/src/superblock.c new file mode 100644 index 00000000..2f35709b --- /dev/null +++ b/src/superblock.c @@ -0,0 +1,283 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +/* + * superblock.c -- + * + * Implementation of the SplinterDB superblock (see superblock.h). + */ + +#include "superblock.h" +#include "poison.h" + +#define SUPERBLOCK_CSUM_SEED (0x53555045524231ULL) // "SUPERB1" + +static uint64 +superblock_slot_addr(const superblock_context *ctx, uint64 slot) +{ + platform_assert(slot < SUPERBLOCK_NUM_SLOTS); + return slot * ctx->page_size; +} + +static checksum128 +superblock_checksum(const superblock *sb) +{ + return platform_checksum128( + sb, offsetof(superblock, checksum), SUPERBLOCK_CSUM_SEED); +} + +static bool32 +superblock_geometry_matches(disk_geometry geometry, const allocator_config *cfg) +{ + return geometry.disk_size == cfg->capacity + && geometry.page_size == cfg->io_cfg->page_size + && geometry.extent_size == cfg->io_cfg->extent_size; +} + +static bool32 +superblock_is_valid(const superblock *sb, const allocator_config *cfg) +{ + // Copy the packed geometry member by value (avoids an unaligned address). + disk_geometry geometry = sb->geometry; + return sb->format_magic == SUPERBLOCK_FORMAT_MAGIC + && sb->format_version == SUPERBLOCK_FORMAT_VERSION + && superblock_geometry_matches(geometry, cfg) + && platform_checksum_is_equal(sb->checksum, + superblock_checksum(sb)); +} + +platform_status +superblock_read_geometry(const char *filename, disk_geometry *geometry) +{ + /* + * Geometry is the first field of the superblock at page 0, and it is + * write-invariant, so a raw read of it is safe even against a torn write of + * the page's mutable fields. + */ + return io_read_bootstrap(filename, geometry, sizeof(*geometry), 0); +} + +platform_status +superblock_context_init(superblock_context *ctx, + io_handle *io, + const allocator_config *cfg, + platform_heap_id hid) +{ + ZERO_CONTENTS(ctx); + ctx->io = io; + ctx->heap_id = hid; + ctx->page_size = cfg->io_cfg->page_size; + + platform_assert(sizeof(superblock) <= ctx->page_size); + + platform_status rc = platform_buffer_init(&ctx->image_buffer, ctx->page_size); + if (!SUCCESS(rc)) { + return rc; + } + ctx->image = platform_buffer_getaddr(&ctx->image_buffer); + memset(ctx->image, 0, ctx->page_size); + ctx->current_slot = 0; + return STATUS_OK; +} + +void +superblock_context_deinit(superblock_context *ctx) +{ + if (ctx->image != NULL) { + platform_buffer_deinit(&ctx->image_buffer); + ctx->image = NULL; + } +} + +platform_status +superblock_mount(superblock_context *ctx, const allocator_config *cfg) +{ + platform_assert(ctx->image != NULL); + + buffer_handle slot_buffer; + platform_status rc = platform_buffer_init(&slot_buffer, ctx->page_size); + if (!SUCCESS(rc)) { + return rc; + } + superblock *slot = platform_buffer_getaddr(&slot_buffer); + + bool32 have_valid = FALSE; + for (uint64 s = 0; s < SUPERBLOCK_NUM_SLOTS; s++) { + rc = io_read( + ctx->io, slot, ctx->page_size, superblock_slot_addr(ctx, s)); + if (!SUCCESS(rc)) { + goto out; + } + if (!superblock_is_valid(slot, cfg)) { + continue; + } + if (!have_valid || slot->generation > ctx->image->generation) { + memcpy(ctx->image, slot, ctx->page_size); + ctx->current_slot = s; + have_valid = TRUE; + } + } + + if (!have_valid) { + platform_error_log("superblock_mount: no valid superblock found\n"); + rc = STATUS_NOT_FOUND; + goto out; + } + rc = STATUS_OK; + +out: + { + platform_status deinit_rc = platform_buffer_deinit(&slot_buffer); + if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { + rc = deinit_rc; + } + } + return rc; +} + +/* + * Checksum the in-memory image and write it to a physical slot. Does not make + * it durable; superblock_publish() issues the barrier. + */ +static platform_status +superblock_write_slot(superblock_context *ctx, uint64 slot) +{ + ctx->image->checksum = superblock_checksum(ctx->image); + return io_write( + ctx->io, ctx->image, ctx->page_size, superblock_slot_addr(ctx, slot)); +} + +platform_status +superblock_publish(superblock_context *ctx) +{ + platform_assert(ctx->image != NULL); + + uint64 target = ctx->current_slot ^ 1; + ctx->image->generation += 1; + platform_assert(ctx->image->generation != 0); // generation wraparound + + platform_status rc = superblock_write_slot(ctx, target); + if (!SUCCESS(rc)) { + return rc; + } + rc = io_durable_barrier(ctx->io); + if (!SUCCESS(rc)) { + return rc; + } + ctx->current_slot = target; + return STATUS_OK; +} + +platform_status +superblock_format(superblock_context *ctx, const allocator_config *cfg) +{ + platform_assert(ctx->image != NULL); + + memset(ctx->image, 0, ctx->page_size); + ctx->image->geometry.disk_size = cfg->capacity; + ctx->image->geometry.page_size = cfg->io_cfg->page_size; + ctx->image->geometry.extent_size = cfg->io_cfg->extent_size; + ctx->image->format_magic = SUPERBLOCK_FORMAT_MAGIC; + ctx->image->format_version = SUPERBLOCK_FORMAT_VERSION; + ctx->image->generation = 0; + ctx->image->allocation_state_addr = 0; // fresh DB: rebuild on crash + for (uint64 i = 0; i < SUPERBLOCK_MAX_TREES; i++) { + ctx->image->trees[i].table_id = INVALID_ALLOCATOR_ROOT_ID; + } + + /* + * Write both physical copies so torn-write protection is in force from the + * outset. The first publish targets slot 0 (generation 1) because the + * initial current_slot is 1; the second targets slot 1 (generation 2), + * leaving slot 1 newest. + */ + ctx->current_slot = 1; + platform_status rc = superblock_publish(ctx); + if (!SUCCESS(rc)) { + return rc; + } + return superblock_publish(ctx); +} + +bool32 +superblock_allocation_state_valid(const superblock_context *ctx) +{ + return ctx->image->allocation_state_addr != 0; +} + +uint64 +superblock_allocation_state_addr(const superblock_context *ctx) +{ + return ctx->image->allocation_state_addr; +} + +void +superblock_set_allocation_state_addr(superblock_context *ctx, uint64 addr) +{ + ctx->image->allocation_state_addr = addr; +} + +platform_status +superblock_get_tree_record(const superblock_context *ctx, + allocator_root_id table_id, + superblock_tree_record *out) +{ + platform_assert(table_id != INVALID_ALLOCATOR_ROOT_ID); + for (uint64 i = 0; i < SUPERBLOCK_MAX_TREES; i++) { + if (ctx->image->trees[i].table_id == table_id) { + *out = ctx->image->trees[i]; + return STATUS_OK; + } + } + return STATUS_NOT_FOUND; +} + +platform_status +superblock_set_tree_record(superblock_context *ctx, + const superblock_tree_record *rec) +{ + platform_assert(rec->table_id != INVALID_ALLOCATOR_ROOT_ID); + + // Overwrite an existing record for this table_id if present. + for (uint64 i = 0; i < SUPERBLOCK_MAX_TREES; i++) { + if (ctx->image->trees[i].table_id == rec->table_id) { + ctx->image->trees[i] = *rec; + return STATUS_OK; + } + } + // Otherwise claim the first free slot. + for (uint64 i = 0; i < SUPERBLOCK_MAX_TREES; i++) { + if (ctx->image->trees[i].table_id == INVALID_ALLOCATOR_ROOT_ID) { + ctx->image->trees[i] = *rec; + return STATUS_OK; + } + } + return STATUS_NO_SPACE; +} + +platform_status +superblock_remove_tree_record(superblock_context *ctx, + allocator_root_id table_id) +{ + platform_assert(table_id != INVALID_ALLOCATOR_ROOT_ID); + for (uint64 i = 0; i < SUPERBLOCK_MAX_TREES; i++) { + if (ctx->image->trees[i].table_id == table_id) { + ZERO_CONTENTS(&ctx->image->trees[i]); + ctx->image->trees[i].table_id = INVALID_ALLOCATOR_ROOT_ID; + return STATUS_OK; + } + } + return STATUS_NOT_FOUND; +} + +uint64 +superblock_num_trees(const superblock_context *ctx) +{ + uint64 count = 0; + for (uint64 i = 0; i < SUPERBLOCK_MAX_TREES; i++) { + if (ctx->image->trees[i].table_id != INVALID_ALLOCATOR_ROOT_ID) { + count++; + } + } + return count; +} diff --git a/src/superblock.h b/src/superblock.h new file mode 100644 index 00000000..487eae46 --- /dev/null +++ b/src/superblock.h @@ -0,0 +1,197 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +/* + * superblock.h -- + * + * The SplinterDB superblock: the single, atomically-updated root of all + * durable instance metadata. It unifies what used to be three unrelated + * mechanisms (the allocator meta page, the allocator clean-state A/B + * records, and core's per-table checkpoint directory + record A/B pairs) + * into one structure, so that a checkpoint advances a tree's root and + * invalidates the persisted allocation state in a single atomic write. + * + * Two physical copies live at pages 0 and 1 of the device. Writes + * alternate between them and carry a monotonic generation number, so a + * torn write always leaves the previous generation intact in the other + * page. Mount picks the newest copy that passes magic/version/checksum/ + * geometry validation. + * + * The disk geometry is the first field (offset 0) so it can be read via a + * raw bootstrap read before any page size is known. Geometry is + * write-invariant -- every generation writes identical geometry bytes -- + * so a torn write to page 0 can only damage the checksummed mutable fields + * (recovered from the other page), never the geometry the bootstrap read + * depends on. + * + * This is intentionally a new on-disk format; older databases must be + * reformatted. + */ + +#pragma once + +#include "platform_hash.h" +#include "platform_buffer.h" +#include "allocator.h" +#include "platform_io.h" +#include "util.h" + +/* + * Format headroom for multiple trees. The current system asserts at most one + * occupied record (see the caller's claim path); the array is sized so that + * multi-tree support becomes a machinery change, not a format change. + */ +#define SUPERBLOCK_MAX_TREES (30) + +#define SUPERBLOCK_FORMAT_MAGIC (0x5344425355504552ULL) // SDBSUPER +#define SUPERBLOCK_FORMAT_VERSION (1) + +/* The two physical superblock copies live at pages 0 and 1. */ +#define SUPERBLOCK_NUM_SLOTS (2) + +/* + * A per-tree durable record. table_id == INVALID_ALLOCATOR_ROOT_ID marks an + * empty slot. log_meta_head is format headroom for the per-tree segmented log + * (the oldest live segment, from whose start recovery replays); it is 0 until + * the log is wired. No intra-segment replay offset is stored: recovery replays + * from a segment boundary and skips records whose generation is at or below + * incorporated_generation. + */ +typedef struct ONDISK superblock_tree_record { + uint64 table_id; + uint64 root_addr; + uint64 log_meta_head; // oldest live log segment = replay start (0 until wired) + /* + * Highest memtable generation folded into root_addr; UINT64_MAX means none + * has been incorporated yet (distinct from generation 0, which is a real, + * live generation). Drives memtable-generation resume at mount and, once + * the log is wired, the replay-skip boundary. + */ + uint64 incorporated_generation; + bool32 unmounted; // TRUE iff root_addr is a clean-unmount root + uint32 pad; // explicit: keep trailing on-disk bytes deterministic +} superblock_tree_record; + +/* Sentinel for superblock_tree_record.incorporated_generation. */ +#define SUPERBLOCK_NO_INCORPORATED_GENERATION (UINT64_MAX) + +typedef struct ONDISK superblock { + disk_geometry geometry; // MUST be first; see the bootstrap-read note above. + uint64 format_magic; + uint64 format_version; + uint64 generation; // monotonic; newest valid copy wins at mount + /* + * Base address of the persisted allocator refcount map when it is + * trustworthy, or 0 to force a rebuild-by-walking on the next mount. A + * checkpoint writes this as 0 in the same atomic update that advances a + * root; a clean unmount writes it nonzero after the map is durable. + */ + uint64 allocation_state_addr; + superblock_tree_record trees[SUPERBLOCK_MAX_TREES]; + checksum128 checksum; +} superblock; + +_Static_assert(offsetof(superblock, geometry) == 0, + "disk geometry must be the first superblock field"); +_Static_assert(sizeof(superblock) <= IO_DEFAULT_PAGE_SIZE, + "superblock must fit in the default page size"); + +/* + * In-memory handle over the on-disk superblock. Holds a page-aligned image of + * the current superblock and tracks which physical slot (0 or 1) it came from, + * so a publish can target the other slot. + */ +typedef struct superblock_context { + io_handle *io; + platform_heap_id heap_id; + uint64 page_size; + buffer_handle image_buffer; + superblock *image; // page-aligned, page-sized + uint64 current_slot; // slot the in-memory image was last read/written +} superblock_context; + +/* + * Read the raw disk geometry via a bootstrap read of page 0, before any + * subsystem is configured. Safe against torn writes because geometry is + * write-invariant. Does not validate the checksum (the full mount does). + */ +platform_status +superblock_read_geometry(const char *filename, disk_geometry *geometry); + +/* Allocate the in-memory image. Does no I/O. */ +platform_status +superblock_context_init(superblock_context *ctx, + io_handle *io, + const allocator_config *cfg, + platform_heap_id hid); + +void +superblock_context_deinit(superblock_context *ctx); + +/* + * Read both physical copies, validate each (magic, version, checksum, and + * geometry against cfg), and load the newest valid one into the in-memory + * image. Returns STATUS_NOT_FOUND if neither copy is a valid superblock. + */ +platform_status +superblock_mount(superblock_context *ctx, const allocator_config *cfg); + +/* + * Initialize a fresh superblock in the in-memory image (empty tree table, + * allocation state invalid) and write both physical copies durably. Used by + * mkfs. + */ +platform_status +superblock_format(superblock_context *ctx, const allocator_config *cfg); + +/* + * Publish the current in-memory image: bump the generation, write it to the + * physical slot not currently newest, and make it durable. This is the single + * atomic commit for a checkpoint or a clean-unmount transition; mutate the + * image first via the setters below (tree records, allocation state), then + * publish. + */ +platform_status +superblock_publish(superblock_context *ctx); + +/* ---- Accessors on the in-memory image ---- */ + +bool32 +superblock_allocation_state_valid(const superblock_context *ctx); + +uint64 +superblock_allocation_state_addr(const superblock_context *ctx); + +void +superblock_set_allocation_state_addr(superblock_context *ctx, uint64 addr); + +/* + * Copy the tree record for table_id into *out. Returns STATUS_NOT_FOUND if + * table_id has no record. + */ +platform_status +superblock_get_tree_record(const superblock_context *ctx, + allocator_root_id table_id, + superblock_tree_record *out); + +/* + * Upsert rec into the in-memory image, matched by rec->table_id. A new + * table_id claims a free slot; STATUS_NO_SPACE if none remain. In-memory + * only; not durable until superblock_publish(). + */ +platform_status +superblock_set_tree_record(superblock_context *ctx, + const superblock_tree_record *rec); + +/* + * Remove the tree record for table_id from the in-memory image, freeing its + * slot. Returns STATUS_NOT_FOUND if table_id has no record. In-memory only; + * not durable until superblock_publish(). + */ +platform_status +superblock_remove_tree_record(superblock_context *ctx, + allocator_root_id table_id); + +/* Number of occupied tree records in the in-memory image. */ +uint64 +superblock_num_trees(const superblock_context *ctx); diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index c5f8a600..14be1a27 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -151,7 +151,8 @@ cache_test_verify_disk_page(cache *cc, * allocator_open_refcounts(). rebuild == FALSE loads the trusted persisted * map; rebuild == TRUE ignores it and reserves only the allocator's own fixed * extents, leaving all other ownership for a caller-driven rebuild. - * persist() writes the map durably; abort_recovery() never persists. This + * persist() writes the map durably; if a rebuild fails there is nothing to + * undo -- deinit() alone discards it, since nothing has been persisted. This * exercises that contract end to end (clean-vs-rebuild is the caller's * decision -- the superblock's -- not the allocator's). */ @@ -263,9 +264,9 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, goto cleanup; } - // 4) Abort never persists: a later clean open still sees the pre-rebuild - // map (the aborted rebuild left no trace on disk). - rc_allocator_abort_recovery(&al); + // 4) A failed/abandoned rebuild persists nothing: deinit alone discards it, + // so a later clean open still sees the pre-rebuild map. + rc_allocator_deinit(&al); al_live = FALSE; rc = rc_allocator_mount(&al, cfg, io, hid, platform_get_module_id()); @@ -342,11 +343,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, cleanup: if (al_live) { - if (al.recovery_in_progress) { - rc_allocator_abort_recovery(&al); - } else { - rc_allocator_deinit(&al); - } + rc_allocator_deinit(&al); } if (SUCCESS(rc)) { @@ -516,11 +513,7 @@ test_mini_recover_allocations(allocator_config *allocator_cfg, clockcache_deinit(&recovery_cc); } if (recovery_al_live) { - if (recovery.recovery_in_progress) { - rc_allocator_abort_recovery(&recovery); - } else { - rc_allocator_deinit(&recovery); - } + rc_allocator_deinit(&recovery); } if (original_cc_live) { clockcache_deinit(&original_cc); diff --git a/tests/unit/superblock_test.c b/tests/unit/superblock_test.c new file mode 100644 index 00000000..8bdeb4ff --- /dev/null +++ b/tests/unit/superblock_test.c @@ -0,0 +1,237 @@ +// Copyright 2018-2026 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +/* + * ----------------------------------------------------------------------------- + * superblock_test.c -- + * + * Unit tests for the unified superblock (src/superblock.c): fresh format + * state, durable publish + re-mount round-trip, the A/B generation selection, + * and torn-write fallback (a corrupted newest slot must leave the previous + * generation intact and mountable). + * ----------------------------------------------------------------------------- + */ +#include "unit_tests.h" +#include "ctest.h" // This is required for all test-case files. +#include "platform.h" +#include "config.h" // Reqd for definition of master_config{} +#include "allocator.h" +#include "superblock.h" + +CTEST_DATA(superblock) +{ + platform_heap_id hid; + io_config io_cfg; + io_handle *ioh; + allocator_config allocator_cfg; +}; + +CTEST_SETUP(superblock) +{ + platform_register_thread(); + + bool use_shmem = config_parse_use_shmem(Ctest_argc, (char **)Ctest_argv); + + platform_status rc = platform_heap_create( + platform_get_module_id(), (256 * MiB), use_shmem, &data->hid); + platform_assert_status_ok(rc); + + master_config master_cfg; + config_set_defaults(&master_cfg); + + io_config_init(&data->io_cfg, + master_cfg.page_size, + master_cfg.extent_size, + master_cfg.io_flags, + master_cfg.io_perms, + master_cfg.io_async_queue_depth, + master_cfg.io_filename); + + data->ioh = io_handle_create(&data->io_cfg, data->hid); + ASSERT_TRUE(data->ioh != NULL, "Failed to create IO handle\n"); + + allocator_config_init( + &data->allocator_cfg, &data->io_cfg, master_cfg.allocator_capacity); +} + +CTEST_TEARDOWN(superblock) +{ + io_handle_destroy(data->ioh); + platform_heap_destroy(&data->hid); + platform_deregister_thread(); +} + +/* Zero one raw superblock slot (page) and make the corruption durable. */ +static void +superblock_test_corrupt_slot(io_handle *io, uint64 page_size, uint64 slot) +{ + buffer_handle buffer; + platform_status rc = platform_buffer_init(&buffer, page_size); + platform_assert_status_ok(rc); + + void *page = platform_buffer_getaddr(&buffer); + memset(page, 0, page_size); + rc = io_write(io, page, page_size, slot * page_size); + platform_assert_status_ok(rc); + rc = io_durable_barrier(io); + platform_assert_status_ok(rc); + + platform_buffer_deinit(&buffer); +} + +/* + * A freshly formatted superblock has an empty tree table and invalid (rebuild) + * allocation state, and a subsequent mount reads that back. + */ +CTEST2(superblock, test_format_sets_fresh_state) +{ + superblock_context ctx; + platform_status rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + rc = superblock_format(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_FALSE(superblock_allocation_state_valid(&ctx)); + ASSERT_EQUAL(0, superblock_num_trees(&ctx)); + superblock_context_deinit(&ctx); + + rc = superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_mount(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_EQUAL(0, superblock_num_trees(&ctx)); + ASSERT_FALSE(superblock_allocation_state_valid(&ctx)); + superblock_context_deinit(&ctx); +} + +/* + * A published tree record and allocation state survive a deinit + re-mount: + * the newest generation wins. + */ +CTEST2(superblock, test_publish_persists_tree_record) +{ + superblock_context ctx; + platform_status rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_format(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + + superblock_tree_record rec = { + .table_id = 1, + .root_addr = 0x4000, + .log_meta_head = 0, + .incorporated_generation = SUPERBLOCK_NO_INCORPORATED_GENERATION, + .unmounted = TRUE, + }; + rc = superblock_set_tree_record(&ctx, &rec); + ASSERT_TRUE(SUCCESS(rc)); + superblock_set_allocation_state_addr(&ctx, 0x8000); + rc = superblock_publish(&ctx); + ASSERT_TRUE(SUCCESS(rc)); + superblock_context_deinit(&ctx); + + rc = superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_mount(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_EQUAL(1, superblock_num_trees(&ctx)); + ASSERT_TRUE(superblock_allocation_state_valid(&ctx)); + ASSERT_EQUAL(0x8000, superblock_allocation_state_addr(&ctx)); + + superblock_tree_record got; + rc = superblock_get_tree_record(&ctx, 1, &got); + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_EQUAL(0x4000, got.root_addr); + ASSERT_TRUE(got.unmounted); + superblock_context_deinit(&ctx); +} + +/* + * The core torn-write guarantee: format writes gen 1 to slot 0 and gen 2 to + * slot 1, so the next publish targets slot 0 (gen 3) and carries the new tree + * record. Corrupting that newest slot must leave slot 1 (gen 2, which predates + * the record) intact, and mount must fall back to it. + */ +CTEST2(superblock, test_torn_write_falls_back_to_older_generation) +{ + superblock_context ctx; + platform_status rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_format(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + + // After format the image is gen 2 in slot 1, so this publish targets slot 0. + superblock_tree_record rec = { + .table_id = 1, + .root_addr = 0x4000, + .incorporated_generation = SUPERBLOCK_NO_INCORPORATED_GENERATION, + .unmounted = TRUE, + }; + rc = superblock_set_tree_record(&ctx, &rec); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_publish(&ctx); + ASSERT_TRUE(SUCCESS(rc)); + superblock_context_deinit(&ctx); + + // Simulate a torn write of the newest slot (slot 0, gen 3). + superblock_test_corrupt_slot(data->ioh, data->io_cfg.page_size, 0); + + // Mount must still succeed, falling back to slot 1 (gen 2) -- which has no + // tree record, proving the older generation was left intact. + rc = superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_mount(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_EQUAL(0, superblock_num_trees(&ctx)); + + superblock_tree_record got; + rc = superblock_get_tree_record(&ctx, 1, &got); + ASSERT_FALSE(SUCCESS(rc)); + superblock_context_deinit(&ctx); +} + +/* Both slots corrupt => no valid superblock. */ +CTEST2(superblock, test_both_slots_corrupt_is_not_found) +{ + superblock_context ctx; + platform_status rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_format(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + superblock_context_deinit(&ctx); + + superblock_test_corrupt_slot(data->ioh, data->io_cfg.page_size, 0); + superblock_test_corrupt_slot(data->ioh, data->io_cfg.page_size, 1); + + rc = superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_mount(&ctx, &data->allocator_cfg); + ASSERT_FALSE(SUCCESS(rc)); + superblock_context_deinit(&ctx); +} + +/* + * The raw bootstrap geometry read (used before any subsystem is configured) + * returns the formatted geometry. + */ +CTEST2(superblock, test_read_geometry) +{ + superblock_context ctx; + platform_status rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_format(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + superblock_context_deinit(&ctx); + + disk_geometry geom; + rc = superblock_read_geometry(data->io_cfg.filename, &geom); + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_EQUAL(data->allocator_cfg.capacity, geom.disk_size); + ASSERT_EQUAL(data->io_cfg.page_size, geom.page_size); + ASSERT_EQUAL(data->io_cfg.extent_size, geom.extent_size); +} From 07fc9397cda4e9486680191ea7fd157be2457c12 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 24 Jul 2026 11:28:24 -0700 Subject: [PATCH 16/64] s/open_refcounts/load_refcounts/g Signed-off-by: Rob Johnson --- src/allocator.h | 10 +++++----- src/core.c | 4 ++-- src/rc_allocator.c | 16 ++++++++-------- src/rc_allocator.h | 8 ++++---- tests/functional/cache_test.c | 20 ++++++++++---------- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/allocator.h b/src/allocator.h index 17fb5c87..d7b7ccb0 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -138,7 +138,7 @@ typedef refcount (*generic_ref_fn)(allocator *al, uint64 addr); /* * Record one logical reference to addr while rebuilding a recovery map (see - * allocator_open_refcounts() with rebuild == TRUE). The first reference to a + * allocator_load_refcounts() with rebuild == TRUE). The first reference to a * given extent establishes its nonzero allocation floor; later references * increment it normally. addr must be the base address of a non-reserved * extent. @@ -158,7 +158,7 @@ typedef platform_status (*recovery_record_reference_fn)(allocator *al, * from durable metadata it owns (the superblock's allocation-state validity), * so the allocator never reads the persisted map when it would be discarded. */ -typedef platform_status (*open_refcounts_fn)(allocator *al, bool32 rebuild); +typedef platform_status (*load_refcounts_fn)(allocator *al, bool32 rebuild); /* * Write the refcount map to its durable location and make it durable. Returns @@ -189,7 +189,7 @@ typedef struct allocator_ops { generic_ref_fn get_ref; recovery_record_reference_fn recovery_record_reference; - open_refcounts_fn open_refcounts; + load_refcounts_fn load_refcounts; persist_refcounts_fn persist; get_size_fn in_use; @@ -245,9 +245,9 @@ allocator_recovery_record_reference(allocator *al, uint64 addr, page_type type) } static inline platform_status -allocator_open_refcounts(allocator *al, bool32 rebuild) +allocator_load_refcounts(allocator *al, bool32 rebuild) { - return al->ops->open_refcounts(al, rebuild); + return al->ops->load_refcounts(al, rebuild); } static inline platform_status diff --git a/src/core.c b/src/core.c index 3250ede0..03137243 100644 --- a/src/core.c +++ b/src/core.c @@ -2104,9 +2104,9 @@ core_mount(core_handle *spl, * precede trunk_snapshot_create_from_addr(), which increments the root's * refcount in the now-loaded map. */ - rc = allocator_open_refcounts(al, rebuild); + rc = allocator_load_refcounts(al, rebuild); if (!SUCCESS(rc)) { - platform_error_log("core_mount: allocator_open_refcounts failed: %s\n", + platform_error_log("core_mount: allocator_load_refcounts failed: %s\n", platform_status_to_string(rc)); goto deinit_superblock; } diff --git a/src/rc_allocator.c b/src/rc_allocator.c index 3e120e78..c222c17c 100644 --- a/src/rc_allocator.c +++ b/src/rc_allocator.c @@ -112,13 +112,13 @@ rc_allocator_recovery_record_reference_virtual(allocator *a, } platform_status -rc_allocator_open_refcounts(rc_allocator *al, bool32 rebuild); +rc_allocator_load_refcounts(rc_allocator *al, bool32 rebuild); platform_status -rc_allocator_open_refcounts_virtual(allocator *a, bool32 rebuild) +rc_allocator_load_refcounts_virtual(allocator *a, bool32 rebuild) { rc_allocator *al = (rc_allocator *)a; - return rc_allocator_open_refcounts(al, rebuild); + return rc_allocator_load_refcounts(al, rebuild); } platform_status @@ -189,7 +189,7 @@ const static allocator_ops rc_allocator_ops = { .dec_ref = rc_allocator_dec_ref_virtual, .get_ref = rc_allocator_get_ref_virtual, .recovery_record_reference = rc_allocator_recovery_record_reference_virtual, - .open_refcounts = rc_allocator_open_refcounts_virtual, + .load_refcounts = rc_allocator_load_refcounts_virtual, .persist = rc_allocator_persist_virtual, .in_use = rc_allocator_in_use_virtual, .get_capacity = rc_allocator_get_capacity_virtual, @@ -444,7 +444,7 @@ rc_allocator_deinit(rc_allocator *al) * Attach an allocator to an existing device: initialize the in-memory * structures and allocate the (zeroed) refcount buffer, but do NOT read * the persisted map. The caller populates the map exactly once via - * allocator_open_refcounts() -- loading the trusted map or initializing a + * allocator_load_refcounts() -- loading the trusted map or initializing a * rebuild -- so a rebuild never pays for a map read it would discard. *---------------------------------------------------------------------- */ @@ -490,10 +490,10 @@ rc_allocator_mount(rc_allocator *al, /* *---------------------------------------------------------------------- - * rc_allocator_open_refcounts -- + * rc_allocator_load_refcounts -- * * Populate the refcount map of an attached allocator (see - * allocator_open_refcounts()). rebuild == FALSE loads the trusted + * allocator_load_refcounts()). rebuild == FALSE loads the trusted * persisted map from its fixed reserved location; rebuild == TRUE * initializes an empty map that reserves only the fixed extents and enters * recovery mode. Geometry and clean-vs-rebuild validity live in the @@ -502,7 +502,7 @@ rc_allocator_mount(rc_allocator *al, *---------------------------------------------------------------------- */ platform_status -rc_allocator_open_refcounts(rc_allocator *al, bool32 rebuild) +rc_allocator_load_refcounts(rc_allocator *al, bool32 rebuild) { platform_assert(al != NULL); platform_assert(al->ref_count != NULL); diff --git a/src/rc_allocator.h b/src/rc_allocator.h index 46996360..88864edc 100644 --- a/src/rc_allocator.h +++ b/src/rc_allocator.h @@ -46,7 +46,7 @@ typedef struct rc_allocator { /* * True once the refcount map is trustworthy: set by a clean - * rc_allocator_open_refcounts(al, rebuild=FALSE) load, or by + * rc_allocator_load_refcounts(al, rebuild=FALSE) load, or by * rc_allocator_rebuild_finish() after a rebuild completes. False from * rc_allocator_mount() (attach) until then, including throughout an * in-progress rebuild. rc_allocator_persist() asserts this is true: an @@ -71,9 +71,9 @@ rc_allocator_deinit(rc_allocator *al); /* * Attach to an existing device: initialize the in-memory structures and the * (zeroed) refcount buffer, but do not read the persisted map. The caller - * then populates the map exactly once via allocator_open_refcounts() (see - * allocator.h): open_refcounts(rebuild=FALSE) loads the trusted persisted map; - * open_refcounts(rebuild=TRUE) initializes an empty map for a rebuild. The + * then populates the map exactly once via allocator_load_refcounts() (see + * allocator.h): load_refcounts(rebuild=FALSE) loads the trusted persisted map; + * load_refcounts(rebuild=TRUE) initializes an empty map for a rebuild. The * caller chooses the mode from the superblock's allocation-state validity, * which it has already read, so a rebuild never pays for a discarded map read. * Before any allocation mutates the loaded map, the caller publishes an diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index 14be1a27..088239dc 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -148,7 +148,7 @@ cache_test_verify_disk_page(cache *cc, /* * The refcount allocator is a pure store: mount() attaches without reading the * persisted map, and the caller then chooses how to populate it via - * allocator_open_refcounts(). rebuild == FALSE loads the trusted persisted + * allocator_load_refcounts(). rebuild == FALSE loads the trusted persisted * map; rebuild == TRUE ignores it and reserves only the allocator's own fixed * extents, leaving all other ownership for a caller-driven rebuild. * persist() writes the map durably; if a rebuild fails there is nothing to @@ -194,13 +194,13 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, rc_allocator_deinit(&al); al_live = FALSE; - // 2) Clean open: attach + open_refcounts(rebuild=FALSE) loads the map. + // 2) Clean open: attach + load_refcounts(rebuild=FALSE) loads the map. rc = rc_allocator_mount(&al, cfg, io, hid, platform_get_module_id()); if (!SUCCESS(rc)) { goto cleanup; } al_live = TRUE; - rc = allocator_open_refcounts((allocator *)&al, FALSE); + rc = allocator_load_refcounts((allocator *)&al, FALSE); if (!SUCCESS(rc)) { goto cleanup; } @@ -215,14 +215,14 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, rc_allocator_deinit(&al); al_live = FALSE; - // 3) Rebuild open: attach + open_refcounts(rebuild=TRUE) ignores the + // 3) Rebuild open: attach + load_refcounts(rebuild=TRUE) ignores the // persisted map, reserving only the fixed extents. rc = rc_allocator_mount(&al, cfg, io, hid, platform_get_module_id()); if (!SUCCESS(rc)) { goto cleanup; } al_live = TRUE; - rc = allocator_open_refcounts((allocator *)&al, TRUE); + rc = allocator_load_refcounts((allocator *)&al, TRUE); if (!SUCCESS(rc)) { goto cleanup; } @@ -274,7 +274,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, goto cleanup; } al_live = TRUE; - rc = allocator_open_refcounts((allocator *)&al, FALSE); + rc = allocator_load_refcounts((allocator *)&al, FALSE); if (!SUCCESS(rc)) { goto cleanup; } @@ -295,7 +295,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, goto cleanup; } al_live = TRUE; - rc = allocator_open_refcounts((allocator *)&al, TRUE); + rc = allocator_load_refcounts((allocator *)&al, TRUE); if (!SUCCESS(rc)) { goto cleanup; } @@ -328,7 +328,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, goto cleanup; } al_live = TRUE; - rc = allocator_open_refcounts((allocator *)&al, FALSE); + rc = allocator_load_refcounts((allocator *)&al, FALSE); if (!SUCCESS(rc)) { goto cleanup; } @@ -438,12 +438,12 @@ test_mini_recover_allocations(allocator_config *allocator_cfg, goto cleanup; } recovery_al_live = TRUE; - rc = allocator_open_refcounts((allocator *)&recovery, TRUE /* rebuild */); + rc = allocator_load_refcounts((allocator *)&recovery, TRUE /* rebuild */); if (!SUCCESS(rc)) { goto cleanup; } - /* open_refcounts(rebuild) deliberately ignores the persisted map. */ + /* load_refcounts(rebuild) deliberately ignores the persisted map. */ if (allocator_get_refcount((allocator *)&recovery, meta_head) != AL_FREE) { platform_error_log( "cache_test: recovery mount trusted a persisted refcount\n"); From 500e4a5c9b44f2b4a5d24ca3b5a66afb7e05f379 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Fri, 24 Jul 2026 23:06:05 -0700 Subject: [PATCH 17/64] cleanup rc_allocator.c Signed-off-by: Rob Johnson --- src/rc_allocator.c | 708 ++++++++++++++++------------------ src/rc_allocator.h | 6 +- tests/functional/cache_test.c | 47 ++- 3 files changed, 354 insertions(+), 407 deletions(-) diff --git a/src/rc_allocator.c b/src/rc_allocator.c index c222c17c..acc84f40 100644 --- a/src/rc_allocator.c +++ b/src/rc_allocator.c @@ -39,165 +39,6 @@ */ #define SHOULD_TRACE(addr) (0) // Do not trace anything -/* - *------------------------------------------------------------------------------ - * Function declarations and virtual trampolines - *------------------------------------------------------------------------------ - */ - -// allocator.h functions - -allocator_config * -rc_allocator_get_config(rc_allocator *al); - -allocator_config * -rc_allocator_get_config_virtual(allocator *a) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_get_config(al); -} - -platform_status -rc_allocator_alloc(rc_allocator *al, uint64 *addr, page_type type); - -platform_status -rc_allocator_alloc_virtual(allocator *a, uint64 *addr, page_type type) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_alloc(al, addr, type); -} - -refcount -rc_allocator_inc_ref(rc_allocator *al, uint64 addr); - -refcount -rc_allocator_inc_ref_virtual(allocator *a, uint64 addr) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_inc_ref(al, addr); -} - -refcount -rc_allocator_dec_ref(rc_allocator *al, uint64 addr, page_type type); - -refcount -rc_allocator_dec_ref_virtual(allocator *a, uint64 addr, page_type type) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_dec_ref(al, addr, type); -} - -refcount -rc_allocator_get_ref(rc_allocator *al, uint64 addr); - -refcount -rc_allocator_get_ref_virtual(allocator *a, uint64 addr) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_get_ref(al, addr); -} - -platform_status -rc_allocator_rebuild_acquire_extent(rc_allocator *al, - uint64 extent_addr, - page_type type); - -platform_status -rc_allocator_recovery_record_reference_virtual(allocator *a, - uint64 addr, - page_type type) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_rebuild_acquire_extent(al, addr, type); -} - -platform_status -rc_allocator_load_refcounts(rc_allocator *al, bool32 rebuild); - -platform_status -rc_allocator_load_refcounts_virtual(allocator *a, bool32 rebuild) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_load_refcounts(al, rebuild); -} - -platform_status -rc_allocator_persist(rc_allocator *al, uint64 *state_addr); - -platform_status -rc_allocator_persist_virtual(allocator *a, uint64 *state_addr) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_persist(al, state_addr); -} - - -uint64 -rc_allocator_in_use(rc_allocator *al); - -uint64 -rc_allocator_in_use_virtual(allocator *a) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_in_use(al); -} - -uint64 -rc_allocator_get_capacity(rc_allocator *al); - -uint64 -rc_allocator_get_capacity_virtual(allocator *a) -{ - rc_allocator *al = (rc_allocator *)a; - return rc_allocator_get_capacity(al); -} - -void -rc_allocator_assert_noleaks(rc_allocator *al); - -void -rc_allocator_assert_noleaks_virtual(allocator *a) -{ - rc_allocator *al = (rc_allocator *)a; - rc_allocator_assert_noleaks(al); -} - -void -rc_allocator_print_stats(rc_allocator *al); - -void -rc_allocator_print_stats_virtual(allocator *a) -{ - rc_allocator *al = (rc_allocator *)a; - rc_allocator_print_stats(al); -} - -void -rc_allocator_print_allocated(rc_allocator *al); - -void -rc_allocator_print_allocated_virtual(allocator *a) -{ - rc_allocator *al = (rc_allocator *)a; - rc_allocator_print_allocated(al); -} - -const static allocator_ops rc_allocator_ops = { - .get_config = rc_allocator_get_config_virtual, - .alloc = rc_allocator_alloc_virtual, - .inc_ref = rc_allocator_inc_ref_virtual, - .dec_ref = rc_allocator_dec_ref_virtual, - .get_ref = rc_allocator_get_ref_virtual, - .recovery_record_reference = rc_allocator_recovery_record_reference_virtual, - .load_refcounts = rc_allocator_load_refcounts_virtual, - .persist = rc_allocator_persist_virtual, - .in_use = rc_allocator_in_use_virtual, - .get_capacity = rc_allocator_get_capacity_virtual, - .assert_noleaks = rc_allocator_assert_noleaks_virtual, - .print_stats = rc_allocator_print_stats_virtual, - .print_allocated = rc_allocator_print_allocated_virtual, -}; - /* * Helper methods */ @@ -362,199 +203,71 @@ rc_allocator_valid_config(allocator_config *cfg) /* *---------------------------------------------------------------------- - * rc_allocator_[de]init -- + * rc_allocator_load_refcounts -- * - * [de]initialize an allocator + * Populate the refcount map of an attached allocator (see + * allocator_load_refcounts()). rebuild == FALSE loads the trusted + * persisted map from its fixed reserved location; rebuild == TRUE + * initializes an empty map that reserves only the fixed extents and enters + * recovery mode. Geometry and clean-vs-rebuild validity live in the + * superblock, which the caller has already read; the allocator only + * touches its own refcount map here. *---------------------------------------------------------------------- */ platform_status -rc_allocator_init(rc_allocator *al, - allocator_config *cfg, - io_handle *io, - platform_heap_id hid, - platform_module_id mid) +rc_allocator_load_refcounts(rc_allocator *al, bool32 rebuild) { - uint64 rc_extent_count; - uint64 addr; - platform_status rc; platform_assert(al != NULL); - ZERO_CONTENTS(al); - al->super.ops = &rc_allocator_ops; - al->cfg = cfg; - al->io = io; - al->heap_id = hid; + platform_assert(al->ref_count != NULL); - rc = rc_allocator_valid_config(cfg); - if (!SUCCESS(rc)) { - return rc; + if (rebuild) { + platform_status status = rc_allocator_recovery_initialize_refcounts(al); + if (!SUCCESS(status)) { + return status; + } + return STATUS_OK; } - rc = platform_mutex_init(&al->lock, mid, al->heap_id); - if (!SUCCESS(rc)) { - platform_error_log("Failed to init mutex for the allocator\n"); - return rc; - } - // To ensure alignment always allocate in multiples of page size. - uint64 buffer_size = rc_allocator_refcount_buffer_size(cfg); - rc = platform_buffer_init(&al->bh, buffer_size); - if (!SUCCESS(rc)) { - platform_mutex_destroy(&al->lock); - platform_error_log("Failed to create buffer for ref counts\n"); - return STATUS_NO_MEMORY; + // Load the trusted refcount map from its fixed reserved location. + uint64 buffer_size = rc_allocator_refcount_buffer_size(al->cfg); + platform_status status = + io_read(al->io, + al->ref_count, + buffer_size, + RC_ALLOCATOR_REFCOUNT_MAP_EXTENT * al->cfg->io_cfg->extent_size); + if (!SUCCESS(status)) { + return status; } - al->ref_count = platform_buffer_getaddr(&al->bh); - memset(al->ref_count, 0, buffer_size); - - // Reserve extent 0 for the superblock; the superblock module owns its - // pages 0/1. - allocator_alloc(&al->super, &addr, PAGE_TYPE_SUPERBLOCK); - platform_assert(addr == 0); - // Reserve the refcount-map extents (extent 1 .. rc_extent_count). - rc_extent_count = rc_allocator_refcount_extent_count(cfg); - for (uint64 i = 0; i < rc_extent_count; i++) { - allocator_alloc(&al->super, &addr, PAGE_TYPE_SUPERBLOCK); - platform_assert(addr == cfg->io_cfg->extent_size * (i + 1)); + // Compute curr_allocated authoritatively from the loaded map (set, not + // accumulate), so this is correct even if the allocator's stats were not + // freshly zeroed. + al->stats.curr_allocated = 0; + for (uint64 i = 0; i < al->cfg->extent_capacity; i++) { + if (al->ref_count[i] != 0) { + al->stats.curr_allocated++; + } } - - /* - * A fresh map is synthesized correctly in memory -- there is nothing to - * load, so it's trustworthy immediately, like a map that just finished a - * rebuild. It is not persisted here, though: the superblock that mkfs - * writes records allocation_state_addr == 0, so a crash before a later - * clean close enters rebuild recovery rather than trusting this in-memory - * map. The map is persisted only by rc_allocator_persist(). - */ al->map_is_valid = TRUE; return STATUS_OK; } -void -rc_allocator_deinit(rc_allocator *al) -{ - platform_buffer_deinit(&al->bh); - al->ref_count = NULL; - platform_mutex_destroy(&al->lock); -} - -/* - *---------------------------------------------------------------------- - * rc_allocator_mount -- - * - * Attach an allocator to an existing device: initialize the in-memory - * structures and allocate the (zeroed) refcount buffer, but do NOT read - * the persisted map. The caller populates the map exactly once via - * allocator_load_refcounts() -- loading the trusted map or initializing a - * rebuild -- so a rebuild never pays for a map read it would discard. - *---------------------------------------------------------------------- - */ platform_status -rc_allocator_mount(rc_allocator *al, - allocator_config *cfg, - io_handle *io, - platform_heap_id hid, - platform_module_id mid) +rc_allocator_recovery_record_reference(rc_allocator *al, + uint64 extent_addr, + page_type type) { - platform_status status; - - platform_assert(al != NULL); - ZERO_CONTENTS(al); - al->super.ops = &rc_allocator_ops; - al->cfg = cfg; - al->io = io; - al->heap_id = hid; + if (al->map_is_valid) { + platform_error_log( + "Cannot acquire allocator extent once the refcount map is valid.\n"); + return STATUS_INVALID_STATE; + } - status = platform_mutex_init(&al->lock, mid, al->heap_id); - if (!SUCCESS(status)) { - platform_error_log("Failed to init mutex for the allocator\n"); - return status; - } - - platform_assert(cfg->io_cfg->page_size % 4096 == 0); - platform_assert(cfg->capacity - == cfg->io_cfg->extent_size * cfg->extent_capacity); - platform_assert(cfg->capacity - == cfg->io_cfg->page_size * cfg->page_capacity); - - uint64 buffer_size = rc_allocator_refcount_buffer_size(cfg); - status = platform_buffer_init(&al->bh, buffer_size); - if (!SUCCESS(status)) { - platform_mutex_destroy(&al->lock); - platform_error_log("Failed to create buffer to load ref counts\n"); - return STATUS_NO_MEMORY; - } - al->ref_count = platform_buffer_getaddr(&al->bh); - memset(al->ref_count, 0, buffer_size); - return STATUS_OK; -} - -/* - *---------------------------------------------------------------------- - * rc_allocator_load_refcounts -- - * - * Populate the refcount map of an attached allocator (see - * allocator_load_refcounts()). rebuild == FALSE loads the trusted - * persisted map from its fixed reserved location; rebuild == TRUE - * initializes an empty map that reserves only the fixed extents and enters - * recovery mode. Geometry and clean-vs-rebuild validity live in the - * superblock, which the caller has already read; the allocator only - * touches its own refcount map here. - *---------------------------------------------------------------------- - */ -platform_status -rc_allocator_load_refcounts(rc_allocator *al, bool32 rebuild) -{ - platform_assert(al != NULL); - platform_assert(al->ref_count != NULL); - - if (rebuild) { - platform_status status = rc_allocator_recovery_initialize_refcounts(al); - if (!SUCCESS(status)) { - return status; - } - return STATUS_OK; - } - - // Load the trusted refcount map from its fixed reserved location. - uint64 buffer_size = rc_allocator_refcount_buffer_size(al->cfg); - platform_status status = io_read(al->io, - al->ref_count, - buffer_size, - RC_ALLOCATOR_REFCOUNT_MAP_EXTENT - * al->cfg->io_cfg->extent_size); - if (!SUCCESS(status)) { - return status; - } - - // Compute curr_allocated authoritatively from the loaded map (set, not - // accumulate), so this is correct even if the allocator's stats were not - // freshly zeroed. - al->stats.curr_allocated = 0; - for (uint64 i = 0; i < al->cfg->extent_capacity; i++) { - if (al->ref_count[i] != 0) { - al->stats.curr_allocated++; - } - } - al->map_is_valid = TRUE; - return STATUS_OK; -} - -platform_status -rc_allocator_rebuild_acquire_extent(rc_allocator *al, - uint64 extent_addr, - page_type type) -{ - if (al->map_is_valid) { - platform_error_log( - "Cannot acquire allocator extent once the refcount map is valid.\n"); - return STATUS_INVALID_STATE; - } - - uint64 extent_size = al->cfg->io_cfg->extent_size; - if (extent_addr >= al->cfg->capacity || extent_addr % extent_size != 0) { - platform_error_log("Invalid allocator recovery extent address %lu.\n", - extent_addr); - return STATUS_BAD_PARAM; + uint64 extent_size = al->cfg->io_cfg->extent_size; + if (extent_addr >= al->cfg->capacity || extent_addr % extent_size != 0) { + platform_error_log("Invalid allocator recovery extent address %lu.\n", + extent_addr); + return STATUS_BAD_PARAM; } uint64 extent_no = rc_allocator_extent_number(al, extent_addr); @@ -566,30 +279,27 @@ rc_allocator_rebuild_acquire_extent(rc_allocator *al, return STATUS_BAD_PARAM; } - while (TRUE) { - refcount old_ref = - __atomic_load_n(&al->ref_count[extent_no], __ATOMIC_RELAXED); - if (old_ref == (refcount)-1) { + + refcount new_refcount = __sync_add_and_fetch(&al->ref_count[extent_no], 1); + if (new_refcount == 0) { + platform_error_log("Allocator recovery refcount overflow for extent " + "%lu.\n", + extent_no); + return STATUS_LIMIT_EXCEEDED; + } else if (new_refcount == AL_NO_REFS) { + new_refcount = __sync_add_and_fetch(&al->ref_count[extent_no], 1); + if (new_refcount == 0) { platform_error_log("Allocator recovery refcount overflow for extent " "%lu.\n", extent_no); return STATUS_LIMIT_EXCEEDED; } - - refcount new_ref = old_ref == AL_FREE ? AL_ONE_REF : old_ref + 1; - if (!__sync_bool_compare_and_swap( - &al->ref_count[extent_no], old_ref, new_ref)) - { - continue; - } - - if (old_ref == AL_FREE) { - platform_assert(type != PAGE_TYPE_INVALID); - rc_allocator_record_allocated_extent(al); - __sync_add_and_fetch(&al->stats.extent_allocs[type], 1); - } - return STATUS_OK; + platform_assert(type != PAGE_TYPE_INVALID); + rc_allocator_record_allocated_extent(al); + __sync_add_and_fetch(&al->stats.extent_allocs[type], 1); } + + return STATUS_OK; } void @@ -822,34 +532,6 @@ rc_allocator_in_use(rc_allocator *al) return al->stats.curr_allocated; } -/* - *---------------------------------------------------------------------- - * rc_allocator_assert_noleaks -- - * - * Asserts that the allocations of each type are completely matched by - * deallocations by operations that do something like create / destroy - * of objects. Primitive function to do some basic cross-checking of - * these operations. - *---------------------------------------------------------------------- - */ -void -rc_allocator_assert_noleaks(rc_allocator *al) -{ - for (page_type type = PAGE_TYPE_FIRST; type < NUM_PAGE_TYPES; type++) { - // Log pages and super-block page are never deallocated. - if ((type == PAGE_TYPE_LOG) || (type == PAGE_TYPE_SUPERBLOCK)) { - continue; - } - if (al->stats.extent_allocs[type] != al->stats.extent_deallocs[type]) { - platform_default_log("assert_noleaks: leak found\n"); - platform_default_log("\n"); - rc_allocator_print_stats(al); - rc_allocator_print_allocated(al); - platform_assert(0); - } - } -} - /* *---------------------------------------------------------------------- * rc_allocator_print_stats() -- @@ -950,3 +632,271 @@ rc_allocator_print_allocated(rc_allocator *al) nextents_found, size_str(nextents_found * extent_size)); } + +/* + *---------------------------------------------------------------------- + * rc_allocator_assert_noleaks -- + * + * Asserts that the allocations of each type are completely matched by + * deallocations by operations that do something like create / destroy + * of objects. Primitive function to do some basic cross-checking of + * these operations. + *---------------------------------------------------------------------- + */ +void +rc_allocator_assert_noleaks(rc_allocator *al) +{ + for (page_type type = PAGE_TYPE_FIRST; type < NUM_PAGE_TYPES; type++) { + // Log pages and super-block page are never deallocated. + if ((type == PAGE_TYPE_LOG) || (type == PAGE_TYPE_SUPERBLOCK)) { + continue; + } + if (al->stats.extent_allocs[type] != al->stats.extent_deallocs[type]) { + platform_default_log("assert_noleaks: leak found\n"); + platform_default_log("\n"); + rc_allocator_print_stats(al); + rc_allocator_print_allocated(al); + platform_assert(0); + } + } +} + +// allocator.h functions + +allocator_config * +rc_allocator_get_config_virtual(allocator *a) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_get_config(al); +} + +platform_status +rc_allocator_alloc_virtual(allocator *a, uint64 *addr, page_type type) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_alloc(al, addr, type); +} + +refcount +rc_allocator_inc_ref_virtual(allocator *a, uint64 addr) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_inc_ref(al, addr); +} + +refcount +rc_allocator_dec_ref_virtual(allocator *a, uint64 addr, page_type type) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_dec_ref(al, addr, type); +} + +refcount +rc_allocator_get_ref_virtual(allocator *a, uint64 addr) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_get_ref(al, addr); +} + +platform_status +rc_allocator_recovery_record_reference_virtual(allocator *a, + uint64 addr, + page_type type) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_recovery_record_reference(al, addr, type); +} + +platform_status +rc_allocator_load_refcounts_virtual(allocator *a, bool32 rebuild) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_load_refcounts(al, rebuild); +} + +platform_status +rc_allocator_persist_virtual(allocator *a, uint64 *state_addr) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_persist(al, state_addr); +} + + +uint64 +rc_allocator_in_use_virtual(allocator *a) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_in_use(al); +} + +uint64 +rc_allocator_get_capacity_virtual(allocator *a) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_get_capacity(al); +} + +void +rc_allocator_assert_noleaks_virtual(allocator *a) +{ + rc_allocator *al = (rc_allocator *)a; + rc_allocator_assert_noleaks(al); +} + +void +rc_allocator_print_stats_virtual(allocator *a) +{ + rc_allocator *al = (rc_allocator *)a; + rc_allocator_print_stats(al); +} + +void +rc_allocator_print_allocated_virtual(allocator *a) +{ + rc_allocator *al = (rc_allocator *)a; + rc_allocator_print_allocated(al); +} + +const static allocator_ops rc_allocator_ops = { + .get_config = rc_allocator_get_config_virtual, + .alloc = rc_allocator_alloc_virtual, + .inc_ref = rc_allocator_inc_ref_virtual, + .dec_ref = rc_allocator_dec_ref_virtual, + .get_ref = rc_allocator_get_ref_virtual, + .recovery_record_reference = rc_allocator_recovery_record_reference_virtual, + .load_refcounts = rc_allocator_load_refcounts_virtual, + .persist = rc_allocator_persist_virtual, + .in_use = rc_allocator_in_use_virtual, + .get_capacity = rc_allocator_get_capacity_virtual, + .assert_noleaks = rc_allocator_assert_noleaks_virtual, + .print_stats = rc_allocator_print_stats_virtual, + .print_allocated = rc_allocator_print_allocated_virtual, +}; + +/* + *---------------------------------------------------------------------- + * rc_allocator_[de]init -- + * + * [de]initialize an allocator + *---------------------------------------------------------------------- + */ +platform_status +rc_allocator_init(rc_allocator *al, + allocator_config *cfg, + io_handle *io, + platform_heap_id hid, + platform_module_id mid) +{ + uint64 rc_extent_count; + uint64 addr; + platform_status rc; + platform_assert(al != NULL); + ZERO_CONTENTS(al); + al->super.ops = &rc_allocator_ops; + al->cfg = cfg; + al->io = io; + al->heap_id = hid; + + rc = rc_allocator_valid_config(cfg); + if (!SUCCESS(rc)) { + return rc; + } + + rc = platform_mutex_init(&al->lock, mid, al->heap_id); + if (!SUCCESS(rc)) { + platform_error_log("Failed to init mutex for the allocator\n"); + return rc; + } + // To ensure alignment always allocate in multiples of page size. + uint64 buffer_size = rc_allocator_refcount_buffer_size(cfg); + rc = platform_buffer_init(&al->bh, buffer_size); + if (!SUCCESS(rc)) { + platform_mutex_destroy(&al->lock); + platform_error_log("Failed to create buffer for ref counts\n"); + return STATUS_NO_MEMORY; + } + al->ref_count = platform_buffer_getaddr(&al->bh); + memset(al->ref_count, 0, buffer_size); + + // Reserve extent 0 for the superblock; the superblock module owns its + // pages 0/1. + allocator_alloc(&al->super, &addr, PAGE_TYPE_SUPERBLOCK); + platform_assert(addr == 0); + + // Reserve the refcount-map extents (extent 1 .. rc_extent_count). + rc_extent_count = rc_allocator_refcount_extent_count(cfg); + for (uint64 i = 0; i < rc_extent_count; i++) { + allocator_alloc(&al->super, &addr, PAGE_TYPE_SUPERBLOCK); + platform_assert(addr == cfg->io_cfg->extent_size * (i + 1)); + } + + /* + * A fresh map is synthesized correctly in memory -- there is nothing to + * load, so it's trustworthy immediately, like a map that just finished a + * rebuild. It is not persisted here, though: the superblock that mkfs + * writes records allocation_state_addr == 0, so a crash before a later + * clean close enters rebuild recovery rather than trusting this in-memory + * map. The map is persisted only by rc_allocator_persist(). + */ + al->map_is_valid = TRUE; + return STATUS_OK; +} + +void +rc_allocator_deinit(rc_allocator *al) +{ + platform_buffer_deinit(&al->bh); + al->ref_count = NULL; + platform_mutex_destroy(&al->lock); +} + +/* + *---------------------------------------------------------------------- + * rc_allocator_mount -- + * + * Attach an allocator to an existing device: initialize the in-memory + * structures and allocate the (zeroed) refcount buffer, but do NOT read + * the persisted map. The caller populates the map exactly once via + * allocator_load_refcounts() -- loading the trusted map or initializing a + * rebuild -- so a rebuild never pays for a map read it would discard. + *---------------------------------------------------------------------- + */ +platform_status +rc_allocator_mount(rc_allocator *al, + allocator_config *cfg, + io_handle *io, + platform_heap_id hid, + platform_module_id mid) +{ + platform_status status; + + platform_assert(al != NULL); + ZERO_CONTENTS(al); + al->super.ops = &rc_allocator_ops; + al->cfg = cfg; + al->io = io; + al->heap_id = hid; + + status = platform_mutex_init(&al->lock, mid, al->heap_id); + if (!SUCCESS(status)) { + platform_error_log("Failed to init mutex for the allocator\n"); + return status; + } + + platform_assert(cfg->io_cfg->page_size % 4096 == 0); + platform_assert(cfg->capacity + == cfg->io_cfg->extent_size * cfg->extent_capacity); + platform_assert(cfg->capacity + == cfg->io_cfg->page_size * cfg->page_capacity); + + uint64 buffer_size = rc_allocator_refcount_buffer_size(cfg); + status = platform_buffer_init(&al->bh, buffer_size); + if (!SUCCESS(status)) { + platform_mutex_destroy(&al->lock); + platform_error_log("Failed to create buffer to load ref counts\n"); + return STATUS_NO_MEMORY; + } + al->ref_count = platform_buffer_getaddr(&al->bh); + memset(al->ref_count, 0, buffer_size); + return STATUS_OK; +} diff --git a/src/rc_allocator.h b/src/rc_allocator.h index 88864edc..beea20d0 100644 --- a/src/rc_allocator.h +++ b/src/rc_allocator.h @@ -94,9 +94,9 @@ rc_allocator_mount(rc_allocator *al, * base address of a non-reserved allocator extent. */ platform_status -rc_allocator_rebuild_acquire_extent(rc_allocator *al, - uint64 extent_addr, - page_type type); +rc_allocator_recovery_record_reference(rc_allocator *al, + uint64 extent_addr, + page_type type); /* * Complete a successful rebuild without performing I/O. A later diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index 088239dc..fce7a5a1 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -169,7 +169,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, // Extent 0 (superblock) + the refcount-map extents. uint64 reserved_extent_count = 1 + refcount_extent_count; uint64 stale_extent_addr = 0; - platform_status rc = STATUS_OK; + platform_status rc = STATUS_OK; rc_allocator al; bool32 al_live = FALSE; @@ -200,7 +200,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, goto cleanup; } al_live = TRUE; - rc = allocator_load_refcounts((allocator *)&al, FALSE); + rc = allocator_load_refcounts((allocator *)&al, FALSE); if (!SUCCESS(rc)) { goto cleanup; } @@ -222,7 +222,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, goto cleanup; } al_live = TRUE; - rc = allocator_load_refcounts((allocator *)&al, TRUE); + rc = allocator_load_refcounts((allocator *)&al, TRUE); if (!SUCCESS(rc)) { goto cleanup; } @@ -252,7 +252,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, } // rebuild_acquire establishes the first reference. - rc = rc_allocator_rebuild_acquire_extent( + rc = rc_allocator_recovery_record_reference( &al, stale_extent_addr, PAGE_TYPE_MISC); if (!SUCCESS(rc) || allocator_get_refcount((allocator *)&al, stale_extent_addr) @@ -274,7 +274,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, goto cleanup; } al_live = TRUE; - rc = allocator_load_refcounts((allocator *)&al, FALSE); + rc = allocator_load_refcounts((allocator *)&al, FALSE); if (!SUCCESS(rc)) { goto cleanup; } @@ -295,16 +295,16 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, goto cleanup; } al_live = TRUE; - rc = allocator_load_refcounts((allocator *)&al, TRUE); + rc = allocator_load_refcounts((allocator *)&al, TRUE); if (!SUCCESS(rc)) { goto cleanup; } - rc = rc_allocator_rebuild_acquire_extent( + rc = rc_allocator_recovery_record_reference( &al, stale_extent_addr, PAGE_TYPE_MISC); if (!SUCCESS(rc)) { goto cleanup; } - rc = rc_allocator_rebuild_acquire_extent( + rc = rc_allocator_recovery_record_reference( &al, stale_extent_addr, PAGE_TYPE_MISC); if (!SUCCESS(rc) || allocator_get_refcount((allocator *)&al, stale_extent_addr) @@ -328,15 +328,14 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, goto cleanup; } al_live = TRUE; - rc = allocator_load_refcounts((allocator *)&al, FALSE); + rc = allocator_load_refcounts((allocator *)&al, FALSE); if (!SUCCESS(rc)) { goto cleanup; } if (allocator_get_refcount((allocator *)&al, stale_extent_addr) != AL_ONE_REF + 1) { - platform_error_log( - "cache_test: finished recovery was not persisted\n"); + platform_error_log("cache_test: finished recovery was not persisted\n"); rc = STATUS_TEST_FAILED; goto cleanup; } @@ -372,17 +371,16 @@ test_mini_recover_allocations(allocator_config *allocator_cfg, platform_heap_id hid) { platform_status rc = STATUS_OK; - rc_allocator original, recovery; - clockcache original_cc, recovery_cc; - bool32 original_al_live = FALSE, original_cc_live = FALSE; - bool32 recovery_al_live = FALSE, recovery_cc_live = FALSE; - mini_allocator mini; - bool32 mini_live = FALSE; - uint64 meta_head = 0; - uint64 data_extents[TEST_MINI_RECOVER_NUM_DATA_EXTENTS] = {0}; + rc_allocator original, recovery; + clockcache original_cc, recovery_cc; + bool32 original_al_live = FALSE, original_cc_live = FALSE; + bool32 recovery_al_live = FALSE, recovery_cc_live = FALSE; + mini_allocator mini; + bool32 mini_live = FALSE; + uint64 meta_head = 0; + uint64 data_extents[TEST_MINI_RECOVER_NUM_DATA_EXTENTS] = {0}; - platform_default_log( - "cache_test: mini_recover_allocations test started\n"); + platform_default_log("cache_test: mini_recover_allocations test started\n"); rc = rc_allocator_init( &original, allocator_cfg, io, hid, platform_get_module_id()); @@ -401,7 +399,7 @@ test_mini_recover_allocations(allocator_config *allocator_cfg, if (!SUCCESS(rc)) { goto cleanup; } - original_cc_live = TRUE; + original_cc_live = TRUE; cache *original_ccp = (cache *)&original_cc; rc = allocator_alloc((allocator *)&original, &meta_head, PAGE_TYPE_MISC); @@ -425,7 +423,7 @@ test_mini_recover_allocations(allocator_config *allocator_cfg, cache_flush(original_ccp); clockcache_deinit(&original_cc); original_cc_live = FALSE; - rc = allocator_persist((allocator *)&original, NULL); + rc = allocator_persist((allocator *)&original, NULL); if (!SUCCESS(rc)) { goto cleanup; } @@ -483,8 +481,7 @@ test_mini_recover_allocations(allocator_config *allocator_cfg, * refcount (which may include self-references beyond this) is explicitly * the job of a higher-level recovery walker, not this function. */ - if (allocator_get_refcount((allocator *)&recovery, meta_head) - != AL_ONE_REF) + if (allocator_get_refcount((allocator *)&recovery, meta_head) != AL_ONE_REF) { platform_error_log( "cache_test: mini_recover_allocations did not recover the metadata " From dbbf4b3b2b6ad580089c00d6a85d286e3c2f8e92 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sat, 25 Jul 2026 00:11:11 -0700 Subject: [PATCH 18/64] cleanup api for loading vs recovering refcounts Signed-off-by: Rob Johnson --- src/allocator.h | 62 +++++++------ src/core.c | 45 +++++---- src/rc_allocator.c | 167 +++++++++++++++++----------------- src/rc_allocator.h | 9 -- tests/functional/cache_test.c | 16 ++-- 5 files changed, 149 insertions(+), 150 deletions(-) diff --git a/src/allocator.h b/src/allocator.h index d7b7ccb0..94b7299b 100644 --- a/src/allocator.h +++ b/src/allocator.h @@ -144,21 +144,13 @@ typedef refcount (*generic_ref_fn)(allocator *al, uint64 addr); * extent. */ typedef platform_status (*recovery_record_reference_fn)(allocator *al, - uint64 addr, - page_type type); + uint64 addr, + page_type type); /* - * Populate the in-memory refcount map of an allocator that has been attached - * (mounted) but whose map is not yet loaded. rebuild == FALSE loads the - * trusted persisted map from its fixed durable location; rebuild == TRUE - * initializes an empty map that reserves only the allocator's own fixed extents - * and enters recovery mode (the caller then rebuilds via - * recovery_record_reference()). Must be called exactly once, after the attach - * mount and before any allocation. The caller learns which mode to request - * from durable metadata it owns (the superblock's allocation-state validity), - * so the allocator never reads the persisted map when it would be discarded. + * */ -typedef platform_status (*load_refcounts_fn)(allocator *al, bool32 rebuild); +typedef platform_status (*generic_status_allocator_fn)(allocator *al); /* * Write the refcount map to its durable location and make it durable. Returns @@ -173,8 +165,7 @@ typedef platform_status (*persist_refcounts_fn)(allocator *al, typedef uint64 (*get_size_fn)(allocator *al); typedef uint64 (*base_addr_fn)(const allocator *al, uint64 addr); -typedef void (*print_fn)(allocator *al); -typedef void (*assert_fn)(allocator *al); +typedef void (*generic_void_allocator_fn)(allocator *al); /* * Define an abstract allocator interface, holding different allocation-related @@ -182,14 +173,20 @@ typedef void (*assert_fn)(allocator *al); */ typedef struct allocator_ops { allocator_get_config_fn get_config; - alloc_fn alloc; - generic_ref_fn inc_ref; - dec_ref_fn dec_ref; - generic_ref_fn get_ref; - recovery_record_reference_fn recovery_record_reference; + alloc_fn alloc; + generic_ref_fn inc_ref; + dec_ref_fn dec_ref; + generic_ref_fn get_ref; + + // After a mount, you must do either a recovery or a load + + generic_status_allocator_fn recovery_begin; + recovery_record_reference_fn recovery_record_reference; + generic_void_allocator_fn recovery_finish; + + generic_status_allocator_fn load_refcounts; - load_refcounts_fn load_refcounts; persist_refcounts_fn persist; get_size_fn in_use; @@ -197,13 +194,13 @@ typedef struct allocator_ops { get_size_fn get_capacity; base_addr_fn extent_base_addr; - assert_fn assert_noleaks; + generic_void_allocator_fn assert_noleaks; - print_fn print_stats; - print_fn print_allocated; + generic_void_allocator_fn print_stats; + generic_void_allocator_fn print_allocated; } allocator_ops; -// To sub-class cache, make a cache your first field; +// To sub-class allocator, make an allocator your first field; struct allocator { const allocator_ops *ops; }; @@ -238,16 +235,29 @@ allocator_get_refcount(allocator *al, uint64 addr) return al->ops->get_ref(al, addr); } +static inline platform_status +allocator_recovery_begin(allocator *al) +{ + return al->ops->recovery_begin(al); +} + static inline platform_status allocator_recovery_record_reference(allocator *al, uint64 addr, page_type type) { return al->ops->recovery_record_reference(al, addr, type); } +static inline void +allocator_recovery_finish(allocator *al) +{ + al->ops->recovery_finish(al); +} + + static inline platform_status -allocator_load_refcounts(allocator *al, bool32 rebuild) +allocator_load_refcounts(allocator *al) { - return al->ops->load_refcounts(al, rebuild); + return al->ops->load_refcounts(al); } static inline platform_status diff --git a/src/core.c b/src/core.c index 03137243..91f7a31c 100644 --- a/src/core.c +++ b/src/core.c @@ -213,13 +213,13 @@ core_publish_root_record(core_handle *spl, bool32 is_unmount) } ZERO_CONTENTS(&rec); - rec.table_id = spl->id; - rec.root_addr = snapshot.root_addr; - rec.log_meta_head = 0; // set once the per-tree log is wired - rec.incorporated_generation = - has_incorporated_generation ? incorporated_generation - : SUPERBLOCK_NO_INCORPORATED_GENERATION; - rec.unmounted = is_unmount; + rec.table_id = spl->id; + rec.root_addr = snapshot.root_addr; + rec.log_meta_head = 0; // set once the per-tree log is wired + rec.incorporated_generation = has_incorporated_generation + ? incorporated_generation + : SUPERBLOCK_NO_INCORPORATED_GENERATION; + rec.unmounted = is_unmount; rc = superblock_set_tree_record(&spl->superblock, &rec); if (!SUCCESS(rc)) { @@ -266,10 +266,11 @@ core_publish_root_record(core_handle *spl, bool32 is_unmount) platform_status release_rc = trunk_snapshot_release(&spl->trunk_context, &old_snapshot); if (!SUCCESS(release_rc)) { - platform_error_log("core_publish_root_record: trunk_snapshot_release " - "failed for old root addr %lu: %s\n", - old_root_addr, - platform_status_to_string(release_rc)); + platform_error_log( + "core_publish_root_record: trunk_snapshot_release " + "failed for old root addr %lu: %s\n", + old_root_addr, + platform_status_to_string(release_rc)); if (SUCCESS(rc)) { rc = release_rc; } @@ -2104,7 +2105,7 @@ core_mount(core_handle *spl, * precede trunk_snapshot_create_from_addr(), which increments the root's * refcount in the now-loaded map. */ - rc = allocator_load_refcounts(al, rebuild); + rc = allocator_load_refcounts(al); if (!SUCCESS(rc)) { platform_error_log("core_mount: allocator_load_refcounts failed: %s\n", platform_status_to_string(rc)); @@ -2118,14 +2119,13 @@ core_mount(core_handle *spl, : rec.incorporated_generation + 1; memtable_config *mt_cfg = &spl->cfg.mt_cfg; - rc = memtable_context_init_at_generation( - &spl->mt_ctxt, - spl->heap_id, - cc, - mt_cfg, - core_memtable_flush_virtual, - spl, - resume_generation); + rc = memtable_context_init_at_generation(&spl->mt_ctxt, + spl->heap_id, + cc, + mt_cfg, + core_memtable_flush_virtual, + spl, + resume_generation); if (!SUCCESS(rc)) { platform_error_log("core_mount: memtable_context_init_at_generation " "failed: %s\n", @@ -2455,9 +2455,8 @@ core_print_super_block(platform_log_handle *log_handle, core_handle *spl) platform_status rc = superblock_get_tree_record(&spl->superblock, spl->id, &rec); if (!SUCCESS(rc)) { - platform_log(log_handle, - "No superblock tree record for root id %lu\n", - spl->id); + platform_log( + log_handle, "No superblock tree record for root id %lu\n", spl->id); return; } diff --git a/src/rc_allocator.c b/src/rc_allocator.c index acc84f40..51cd6c36 100644 --- a/src/rc_allocator.c +++ b/src/rc_allocator.c @@ -101,8 +101,18 @@ rc_allocator_record_allocated_extent(rc_allocator *al) } static platform_status -rc_allocator_recovery_initialize_refcounts(rc_allocator *al) +rc_allocator_init_refcounts(rc_allocator *al) { + uint64 buffer_size = rc_allocator_refcount_buffer_size(al->cfg); + platform_status status = platform_buffer_init(&al->bh, buffer_size); + if (!SUCCESS(status)) { + platform_mutex_destroy(&al->lock); + platform_error_log("Failed to create buffer to load ref counts\n"); + return STATUS_NO_MEMORY; + } + al->ref_count = platform_buffer_getaddr(&al->bh); + memset(al->ref_count, 0, buffer_size); + uint64 reserved_extent_count = rc_allocator_reserved_extent_count(al->cfg); if (reserved_extent_count > al->cfg->extent_capacity) { @@ -200,56 +210,10 @@ rc_allocator_valid_config(allocator_config *cfg) return rc; } - -/* - *---------------------------------------------------------------------- - * rc_allocator_load_refcounts -- - * - * Populate the refcount map of an attached allocator (see - * allocator_load_refcounts()). rebuild == FALSE loads the trusted - * persisted map from its fixed reserved location; rebuild == TRUE - * initializes an empty map that reserves only the fixed extents and enters - * recovery mode. Geometry and clean-vs-rebuild validity live in the - * superblock, which the caller has already read; the allocator only - * touches its own refcount map here. - *---------------------------------------------------------------------- - */ platform_status -rc_allocator_load_refcounts(rc_allocator *al, bool32 rebuild) +rc_allocator_recovery_begin(rc_allocator *al) { - platform_assert(al != NULL); - platform_assert(al->ref_count != NULL); - - if (rebuild) { - platform_status status = rc_allocator_recovery_initialize_refcounts(al); - if (!SUCCESS(status)) { - return status; - } - return STATUS_OK; - } - - // Load the trusted refcount map from its fixed reserved location. - uint64 buffer_size = rc_allocator_refcount_buffer_size(al->cfg); - platform_status status = - io_read(al->io, - al->ref_count, - buffer_size, - RC_ALLOCATOR_REFCOUNT_MAP_EXTENT * al->cfg->io_cfg->extent_size); - if (!SUCCESS(status)) { - return status; - } - - // Compute curr_allocated authoritatively from the loaded map (set, not - // accumulate), so this is correct even if the allocator's stats were not - // freshly zeroed. - al->stats.curr_allocated = 0; - for (uint64 i = 0; i < al->cfg->extent_capacity; i++) { - if (al->ref_count[i] != 0) { - al->stats.curr_allocated++; - } - } - al->map_is_valid = TRUE; - return STATUS_OK; + return rc_allocator_init_refcounts(al); } platform_status @@ -303,7 +267,7 @@ rc_allocator_recovery_record_reference(rc_allocator *al, } void -rc_allocator_rebuild_finish(rc_allocator *al) +rc_allocator_recovery_finish(rc_allocator *al) { platform_assert(al != NULL); platform_assert(!al->map_is_valid); @@ -317,6 +281,49 @@ rc_allocator_rebuild_finish(rc_allocator *al) } +/* + *---------------------------------------------------------------------- + * rc_allocator_load_refcounts -- + * + * Populate the refcount map of an attached allocator (see + * allocator_load_refcounts()). rebuild == FALSE loads the trusted + * persisted map from its fixed reserved location; rebuild == TRUE + * initializes an empty map that reserves only the fixed extents and enters + * recovery mode. Geometry and clean-vs-rebuild validity live in the + * superblock, which the caller has already read; the allocator only + * touches its own refcount map here. + *---------------------------------------------------------------------- + */ +platform_status +rc_allocator_load_refcounts(rc_allocator *al) +{ + platform_assert(al != NULL); + platform_assert(al->ref_count != NULL); + + // Load the trusted refcount map from its fixed reserved location. + uint64 buffer_size = rc_allocator_refcount_buffer_size(al->cfg); + platform_status status = + io_read(al->io, + al->ref_count, + buffer_size, + RC_ALLOCATOR_REFCOUNT_MAP_EXTENT * al->cfg->io_cfg->extent_size); + if (!SUCCESS(status)) { + return status; + } + + // Compute curr_allocated authoritatively from the loaded map (set, not + // accumulate), so this is correct even if the allocator's stats were not + // freshly zeroed. + al->stats.curr_allocated = 0; + for (uint64 i = 0; i < al->cfg->extent_capacity; i++) { + if (al->ref_count[i] != 0) { + al->stats.curr_allocated++; + } + } + al->map_is_valid = TRUE; + return STATUS_OK; +} + /* *---------------------------------------------------------------------- * rc_allocator_persist -- @@ -698,6 +705,13 @@ rc_allocator_get_ref_virtual(allocator *a, uint64 addr) return rc_allocator_get_ref(al, addr); } +platform_status +rc_allocator_recovery_begin_virtual(allocator *a) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_recovery_begin(al); +} + platform_status rc_allocator_recovery_record_reference_virtual(allocator *a, uint64 addr, @@ -707,11 +721,19 @@ rc_allocator_recovery_record_reference_virtual(allocator *a, return rc_allocator_recovery_record_reference(al, addr, type); } +void +rc_allocator_recovery_finish_virtual(allocator *a) +{ + rc_allocator *al = (rc_allocator *)a; + return rc_allocator_recovery_finish(al); +} + + platform_status -rc_allocator_load_refcounts_virtual(allocator *a, bool32 rebuild) +rc_allocator_load_refcounts_virtual(allocator *a) { rc_allocator *al = (rc_allocator *)a; - return rc_allocator_load_refcounts(al, rebuild); + return rc_allocator_load_refcounts(al); } platform_status @@ -763,7 +785,9 @@ const static allocator_ops rc_allocator_ops = { .inc_ref = rc_allocator_inc_ref_virtual, .dec_ref = rc_allocator_dec_ref_virtual, .get_ref = rc_allocator_get_ref_virtual, + .recovery_begin = rc_allocator_recovery_begin_virtual, .recovery_record_reference = rc_allocator_recovery_record_reference_virtual, + .recovery_finish = rc_allocator_recovery_finish_virtual, .load_refcounts = rc_allocator_load_refcounts_virtual, .persist = rc_allocator_persist_virtual, .in_use = rc_allocator_in_use_virtual, @@ -787,8 +811,6 @@ rc_allocator_init(rc_allocator *al, platform_heap_id hid, platform_module_id mid) { - uint64 rc_extent_count; - uint64 addr; platform_status rc; platform_assert(al != NULL); ZERO_CONTENTS(al); @@ -807,27 +829,11 @@ rc_allocator_init(rc_allocator *al, platform_error_log("Failed to init mutex for the allocator\n"); return rc; } - // To ensure alignment always allocate in multiples of page size. - uint64 buffer_size = rc_allocator_refcount_buffer_size(cfg); - rc = platform_buffer_init(&al->bh, buffer_size); + + rc = rc_allocator_init_refcounts(al); if (!SUCCESS(rc)) { platform_mutex_destroy(&al->lock); - platform_error_log("Failed to create buffer for ref counts\n"); - return STATUS_NO_MEMORY; - } - al->ref_count = platform_buffer_getaddr(&al->bh); - memset(al->ref_count, 0, buffer_size); - - // Reserve extent 0 for the superblock; the superblock module owns its - // pages 0/1. - allocator_alloc(&al->super, &addr, PAGE_TYPE_SUPERBLOCK); - platform_assert(addr == 0); - - // Reserve the refcount-map extents (extent 1 .. rc_extent_count). - rc_extent_count = rc_allocator_refcount_extent_count(cfg); - for (uint64 i = 0; i < rc_extent_count; i++) { - allocator_alloc(&al->super, &addr, PAGE_TYPE_SUPERBLOCK); - platform_assert(addr == cfg->io_cfg->extent_size * (i + 1)); + return rc; } /* @@ -845,8 +851,10 @@ rc_allocator_init(rc_allocator *al, void rc_allocator_deinit(rc_allocator *al) { - platform_buffer_deinit(&al->bh); - al->ref_count = NULL; + if (al->ref_count) { + platform_buffer_deinit(&al->bh); + al->ref_count = NULL; + } platform_mutex_destroy(&al->lock); } @@ -889,14 +897,5 @@ rc_allocator_mount(rc_allocator *al, platform_assert(cfg->capacity == cfg->io_cfg->page_size * cfg->page_capacity); - uint64 buffer_size = rc_allocator_refcount_buffer_size(cfg); - status = platform_buffer_init(&al->bh, buffer_size); - if (!SUCCESS(status)) { - platform_mutex_destroy(&al->lock); - platform_error_log("Failed to create buffer to load ref counts\n"); - return STATUS_NO_MEMORY; - } - al->ref_count = platform_buffer_getaddr(&al->bh); - memset(al->ref_count, 0, buffer_size); return STATUS_OK; } diff --git a/src/rc_allocator.h b/src/rc_allocator.h index beea20d0..b155e7d0 100644 --- a/src/rc_allocator.h +++ b/src/rc_allocator.h @@ -97,12 +97,3 @@ platform_status rc_allocator_recovery_record_reference(rc_allocator *al, uint64 extent_addr, page_type type); - -/* - * Complete a successful rebuild without performing I/O. A later - * rc_allocator_persist() may persist the rebuilt table on clean shutdown. If - * the rebuild fails instead, there is nothing to undo: just - * rc_allocator_deinit() the allocator, since nothing has been persisted. - */ -void -rc_allocator_rebuild_finish(rc_allocator *al); diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index fce7a5a1..8a42b411 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -200,7 +200,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, goto cleanup; } al_live = TRUE; - rc = allocator_load_refcounts((allocator *)&al, FALSE); + rc = allocator_load_refcounts((allocator *)&al); if (!SUCCESS(rc)) { goto cleanup; } @@ -222,7 +222,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, goto cleanup; } al_live = TRUE; - rc = allocator_load_refcounts((allocator *)&al, TRUE); + rc = allocator_recovery_begin((allocator *)&al); if (!SUCCESS(rc)) { goto cleanup; } @@ -274,7 +274,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, goto cleanup; } al_live = TRUE; - rc = allocator_load_refcounts((allocator *)&al, FALSE); + rc = allocator_load_refcounts((allocator *)&al); if (!SUCCESS(rc)) { goto cleanup; } @@ -295,7 +295,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, goto cleanup; } al_live = TRUE; - rc = allocator_load_refcounts((allocator *)&al, TRUE); + rc = allocator_recovery_begin((allocator *)&al); if (!SUCCESS(rc)) { goto cleanup; } @@ -315,7 +315,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, rc = STATUS_TEST_FAILED; goto cleanup; } - rc_allocator_rebuild_finish(&al); + allocator_recovery_finish((allocator *)&al); rc = allocator_persist((allocator *)&al, NULL); if (!SUCCESS(rc)) { goto cleanup; @@ -328,7 +328,7 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, goto cleanup; } al_live = TRUE; - rc = allocator_load_refcounts((allocator *)&al, FALSE); + rc = allocator_load_refcounts((allocator *)&al); if (!SUCCESS(rc)) { goto cleanup; } @@ -436,7 +436,7 @@ test_mini_recover_allocations(allocator_config *allocator_cfg, goto cleanup; } recovery_al_live = TRUE; - rc = allocator_load_refcounts((allocator *)&recovery, TRUE /* rebuild */); + rc = allocator_recovery_begin((allocator *)&recovery); if (!SUCCESS(rc)) { goto cleanup; } @@ -500,7 +500,7 @@ test_mini_recover_allocations(allocator_config *allocator_cfg, goto cleanup; } } - rc_allocator_rebuild_finish(&recovery); + allocator_recovery_finish((allocator *)&recovery); cleanup: if (mini_live) { From c54c04ebea87dfa197be6a4bbaaa86a0a2d1e9f7 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sat, 25 Jul 2026 00:15:48 -0700 Subject: [PATCH 19/64] cleanup allocator Signed-off-by: Rob Johnson --- src/rc_allocator.h | 24 +++--------------------- tests/functional/cache_test.c | 12 ++++++------ 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/src/rc_allocator.h b/src/rc_allocator.h index b155e7d0..91eccb72 100644 --- a/src/rc_allocator.h +++ b/src/rc_allocator.h @@ -69,15 +69,9 @@ void rc_allocator_deinit(rc_allocator *al); /* - * Attach to an existing device: initialize the in-memory structures and the - * (zeroed) refcount buffer, but do not read the persisted map. The caller - * then populates the map exactly once via allocator_load_refcounts() (see - * allocator.h): load_refcounts(rebuild=FALSE) loads the trusted persisted map; - * load_refcounts(rebuild=TRUE) initializes an empty map for a rebuild. The - * caller chooses the mode from the superblock's allocation-state validity, - * which it has already read, so a rebuild never pays for a discarded map read. - * Before any allocation mutates the loaded map, the caller publishes an - * invalidated superblock so a crash forces a rebuild. + * Attach to an existing device but do not read the persisted map. The caller + * then populates the map exactly once via allocator_load_refcounts() or + * allocator_recovery_begin/finish. */ platform_status rc_allocator_mount(rc_allocator *al, @@ -85,15 +79,3 @@ rc_allocator_mount(rc_allocator *al, io_handle *io, platform_heap_id hid, platform_module_id mid); - -/* - * Add one logical ownership reference for an extent while rebuilding a - * recovery map. The first reference establishes the allocator's nonzero - * allocation floor (AL_ONE_REF) and records it in stats.extent_allocs[type]; - * later references increment the refcount normally. extent_addr must be the - * base address of a non-reserved allocator extent. - */ -platform_status -rc_allocator_recovery_record_reference(rc_allocator *al, - uint64 extent_addr, - page_type type); diff --git a/tests/functional/cache_test.c b/tests/functional/cache_test.c index 8a42b411..e774f90a 100644 --- a/tests/functional/cache_test.c +++ b/tests/functional/cache_test.c @@ -252,8 +252,8 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, } // rebuild_acquire establishes the first reference. - rc = rc_allocator_recovery_record_reference( - &al, stale_extent_addr, PAGE_TYPE_MISC); + rc = allocator_recovery_record_reference( + (allocator *)&al, stale_extent_addr, PAGE_TYPE_MISC); if (!SUCCESS(rc) || allocator_get_refcount((allocator *)&al, stale_extent_addr) != AL_ONE_REF) @@ -299,13 +299,13 @@ test_rc_allocator_recovery_bootstrap(allocator_config *cfg, if (!SUCCESS(rc)) { goto cleanup; } - rc = rc_allocator_recovery_record_reference( - &al, stale_extent_addr, PAGE_TYPE_MISC); + rc = allocator_recovery_record_reference( + (allocator *)&al, stale_extent_addr, PAGE_TYPE_MISC); if (!SUCCESS(rc)) { goto cleanup; } - rc = rc_allocator_recovery_record_reference( - &al, stale_extent_addr, PAGE_TYPE_MISC); + rc = allocator_recovery_record_reference( + (allocator *)&al, stale_extent_addr, PAGE_TYPE_MISC); if (!SUCCESS(rc) || allocator_get_refcount((allocator *)&al, stale_extent_addr) != AL_ONE_REF + 1) From 26458818d6675b90cfebe0c5341f39f8c047c1a9 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sat, 25 Jul 2026 15:45:14 -0700 Subject: [PATCH 20/64] cleanup superblock Signed-off-by: Rob Johnson --- src/core.c | 100 ++++++++++++----------------------- src/rc_allocator.c | 65 ++++++++++++++++------- src/superblock.c | 65 +++-------------------- src/superblock.h | 69 ++++++++---------------- tests/unit/superblock_test.c | 34 ++++++------ 5 files changed, 127 insertions(+), 206 deletions(-) diff --git a/src/core.c b/src/core.c index 91f7a31c..2282bad9 100644 --- a/src/core.c +++ b/src/core.c @@ -157,15 +157,15 @@ core_capture_checkpoint_cut(core_handle *spl, /* * Publish a tree-record root advance to the superblock: capture the current - * COW root, make it durable, record it (with the incorporated-generation cut - * and the unmounted flag), invalidate the persisted allocation state in the - * same atomic update, and release the previously published root. Used by mkfs - * (empty root) and by unmount (Part A; unmount then persists the map and - * republishes a valid allocation state). The snapshot's owned reference is - * transferred to the durable record on a successful publish. + * COW root, make it durable, record it (with the incorporated-generation cut), + * invalidate the persisted allocation state in the same atomic update, and + * release the previously published root. Used by mkfs (empty root) and by + * unmount (Part A; unmount then persists the map and republishes a valid + * allocation state). The snapshot's owned reference is transferred to the + * durable record on a successful publish. */ static platform_status -core_publish_root_record(core_handle *spl, bool32 is_unmount) +core_publish_root_record(core_handle *spl) { platform_status rc; trunk_snapshot snapshot; @@ -207,24 +207,17 @@ core_publish_root_record(core_handle *spl, bool32 is_unmount) } // The previously published root, retained until the new one is durable. - if (SUCCESS(superblock_get_tree_record(&spl->superblock, spl->id, &old_rec))) - { - old_root_addr = old_rec.root_addr; - } + superblock_get_tree_record(&spl->superblock, &old_rec); + old_root_addr = old_rec.root_addr; ZERO_CONTENTS(&rec); - rec.table_id = spl->id; rec.root_addr = snapshot.root_addr; rec.log_meta_head = 0; // set once the per-tree log is wired rec.incorporated_generation = has_incorporated_generation ? incorporated_generation : SUPERBLOCK_NO_INCORPORATED_GENERATION; - rec.unmounted = is_unmount; - rc = superblock_set_tree_record(&spl->superblock, &rec); - if (!SUCCESS(rc)) { - goto release_snapshot; - } + superblock_set_tree_record(&spl->superblock, &rec); /* * Advancing a root invalidates the persisted allocation map: the in-memory * map now diverges from disk. A clean unmount republishes a valid @@ -2006,7 +1999,7 @@ core_mkfs(core_handle *spl, } // Establish the initial (empty) tree record; publish it durably. - rc = core_publish_root_record(spl, FALSE); + rc = core_publish_root_record(spl); if (!SUCCESS(rc)) { platform_error_log("core_mkfs: core_publish_root_record failed: %s\n", platform_status_to_string(rc)); @@ -2079,21 +2072,18 @@ core_mount(core_handle *spl, } superblock_tree_record rec; - rc = superblock_get_tree_record(&spl->superblock, spl->id, &rec); - if (!SUCCESS(rc)) { - platform_error_log("core_mount: no tree record for root id %lu\n", - spl->id); - goto deinit_superblock; - } + superblock_get_tree_record(&spl->superblock, &rec); /* * Preserve the historical clean-only mount rule for this first format * slice: crash recovery (log replay + allocator rebuild) is not wired yet, - * so only a clean unmount -- one whose tree record is flagged unmounted and - * whose allocation state is still valid -- may supply the root. + * so only a clean at-rest instance -- one whose allocation state is still + * valid -- may supply the root. A valid allocation state is published only + * at the end of a clean unmount, so it is the single at-rest signal (no + * separate per-tree clean flag is needed; see superblock.h). */ bool32 rebuild = !superblock_allocation_state_valid(&spl->superblock); - if (rebuild || !rec.unmounted) { + if (rebuild) { platform_error_log("core_mount: root id %lu requires crash recovery\n", spl->id); rc = STATUS_INVALID_STATE; @@ -2172,17 +2162,12 @@ core_mount(core_handle *spl, } /* - * Mark dirty: flip the tree record to not-unmounted and invalidate the - * persisted allocation state, atomically, before any allocation diverges - * the in-memory map from disk. A crash after this forces the next mount - * into recovery instead of silently reverting to this now-stale root. This - * keeps the same root (no snapshot capture, no refcount change). + * Mark dirty: invalidate the persisted allocation state before any + * allocation diverges the in-memory map from disk. A crash after this + * forces the next mount into recovery instead of silently reverting to this + * now-stale root. The tree record itself is unchanged (same root), so only + * the allocation state is republished. */ - rec.unmounted = FALSE; - rc = superblock_set_tree_record(&spl->superblock, &rec); - if (!SUCCESS(rc)) { - goto deinit_stats; - } superblock_set_allocation_state_addr(&spl->superblock, 0); rc = superblock_publish(&spl->superblock); if (!SUCCESS(rc)) { @@ -2322,7 +2307,7 @@ core_unmount(core_handle *spl) * Part A: publish the clean-unmount root. Allocation state stays invalid * here; it becomes valid only in Part B, after the map is persisted. */ - rc = core_publish_root_record(spl, TRUE); + rc = core_publish_root_record(spl); if (!SUCCESS(rc)) { platform_error_log("core_unmount: failed to publish unmount root: %s\n", platform_status_to_string(rc)); @@ -2379,9 +2364,8 @@ core_destroy(core_handle *spl) * trunk_context_deinit(): trunk_snapshot_release() needs the live context. */ superblock_tree_record rec; - if (SUCCESS(superblock_get_tree_record(&spl->superblock, spl->id, &rec)) - && rec.root_addr != 0) - { + superblock_get_tree_record(&spl->superblock, &rec); + if (rec.root_addr != 0) { trunk_snapshot old_snapshot = {.root_addr = rec.root_addr}; platform_status rc = trunk_snapshot_release(&spl->trunk_context, &old_snapshot); @@ -2396,20 +2380,13 @@ core_destroy(core_handle *spl) core_teardown_after_shutdown(spl); trunk_context_deinit(&spl->trunk_context); - // Erase this tree's record and persist a clean, valid allocation state. - superblock_remove_tree_record(&spl->superblock, spl->id); - uint64 map_addr; - platform_status prc = allocator_persist(spl->al, &map_addr); - if (SUCCESS(prc)) { - superblock_set_allocation_state_addr(&spl->superblock, map_addr); - prc = superblock_publish(&spl->superblock); - } - if (!SUCCESS(prc)) { - platform_error_log("core_destroy: failed to persist allocation state: " - "%s\n", - platform_status_to_string(prc)); - } - + /* + * A destroyed instance must not be reopened. The mount-time mark-dirty + * already left the on-disk allocation state invalid, so we deliberately do + * not persist the map or publish a valid state here: the freed root and its + * subtree stay unreachable, and the next mount rejects the device + * (allocation state invalid) rather than trusting a now-freed root. + */ superblock_context_deinit(&spl->superblock); core_destroy_stats(spl); core_checkpoint_lock_deinit(spl); @@ -2452,25 +2429,18 @@ void core_print_super_block(platform_log_handle *log_handle, core_handle *spl) { superblock_tree_record rec; - platform_status rc = - superblock_get_tree_record(&spl->superblock, spl->id, &rec); - if (!SUCCESS(rc)) { - platform_log( - log_handle, "No superblock tree record for root id %lu\n", spl->id); - return; - } + superblock_get_tree_record(&spl->superblock, &rec); platform_log(log_handle, "Superblock tree record root_id=%lu {\n" " root_addr=%lu log_meta_head=%lu\n" - " incorporated_generation=%lu unmounted=%d\n" + " incorporated_generation=%lu\n" " allocation_state: %s (addr=%lu)\n" "}\n\n", - rec.table_id, + spl->id, rec.root_addr, rec.log_meta_head, rec.incorporated_generation, - rec.unmounted, superblock_allocation_state_valid(&spl->superblock) ? "valid" : "invalid", superblock_allocation_state_addr(&spl->superblock)); diff --git a/src/rc_allocator.c b/src/rc_allocator.c index 51cd6c36..2bedb5c0 100644 --- a/src/rc_allocator.c +++ b/src/rc_allocator.c @@ -100,18 +100,39 @@ rc_allocator_record_allocated_extent(rc_allocator *al) } } +/* + * Allocate the (zeroed) in-memory refcount buffer, if it is not already + * allocated. mkfs (rc_allocator_init), a clean load (rc_allocator_load_ + * refcounts), and a recovery rebuild (rc_allocator_recovery_begin) each ensure + * the buffer exists before populating it. Attach (rc_allocator_mount) + * deliberately does NOT: it leaves ref_count NULL so that using the allocator + * before a load/recovery faults immediately. Idempotent so it tolerates an + * allocator reused across mount cycles without re-attaching. + */ static platform_status -rc_allocator_init_refcounts(rc_allocator *al) +rc_allocator_alloc_refcount_buffer(rc_allocator *al) { + if (al->ref_count != NULL) { + return STATUS_OK; + } uint64 buffer_size = rc_allocator_refcount_buffer_size(al->cfg); platform_status status = platform_buffer_init(&al->bh, buffer_size); if (!SUCCESS(status)) { - platform_mutex_destroy(&al->lock); - platform_error_log("Failed to create buffer to load ref counts\n"); - return STATUS_NO_MEMORY; + platform_error_log("Failed to create buffer for ref counts\n"); + return status; } al->ref_count = platform_buffer_getaddr(&al->bh); memset(al->ref_count, 0, buffer_size); + return STATUS_OK; +} + +static platform_status +rc_allocator_init_refcounts(rc_allocator *al) +{ + platform_status status = rc_allocator_alloc_refcount_buffer(al); + if (!SUCCESS(status)) { + return status; + } uint64 reserved_extent_count = rc_allocator_reserved_extent_count(al->cfg); @@ -285,24 +306,27 @@ rc_allocator_recovery_finish(rc_allocator *al) *---------------------------------------------------------------------- * rc_allocator_load_refcounts -- * - * Populate the refcount map of an attached allocator (see - * allocator_load_refcounts()). rebuild == FALSE loads the trusted - * persisted map from its fixed reserved location; rebuild == TRUE - * initializes an empty map that reserves only the fixed extents and enters - * recovery mode. Geometry and clean-vs-rebuild validity live in the - * superblock, which the caller has already read; the allocator only - * touches its own refcount map here. + * Populate an attached allocator's refcount map by loading the trusted + * persisted map from its fixed reserved location (the clean-mount path; + * see allocator_load_refcounts()). Allocates the refcount buffer that + * attach (rc_allocator_mount) deliberately left NULL. The caller has + * already confirmed via the superblock that the persisted map is valid; + * the allocator only touches its own map here. *---------------------------------------------------------------------- */ platform_status rc_allocator_load_refcounts(rc_allocator *al) { platform_assert(al != NULL); - platform_assert(al->ref_count != NULL); + + platform_status status = rc_allocator_alloc_refcount_buffer(al); + if (!SUCCESS(status)) { + return status; + } // Load the trusted refcount map from its fixed reserved location. - uint64 buffer_size = rc_allocator_refcount_buffer_size(al->cfg); - platform_status status = + uint64 buffer_size = rc_allocator_refcount_buffer_size(al->cfg); + status = io_read(al->io, al->ref_count, buffer_size, @@ -863,10 +887,12 @@ rc_allocator_deinit(rc_allocator *al) * rc_allocator_mount -- * * Attach an allocator to an existing device: initialize the in-memory - * structures and allocate the (zeroed) refcount buffer, but do NOT read - * the persisted map. The caller populates the map exactly once via - * allocator_load_refcounts() -- loading the trusted map or initializing a - * rebuild -- so a rebuild never pays for a map read it would discard. + * structures only. The refcount buffer is deliberately left NULL so that + * using the allocator before its map is populated faults immediately; the + * caller MUST next call rc_allocator_load_refcounts() (clean mount) or + * rc_allocator_recovery_begin() (rebuild), each of which allocates and + * populates the buffer. A rebuild therefore never reads a map it would + * discard. *---------------------------------------------------------------------- */ platform_status @@ -897,5 +923,8 @@ rc_allocator_mount(rc_allocator *al, platform_assert(cfg->capacity == cfg->io_cfg->page_size * cfg->page_capacity); + // Attach only: ref_count is left NULL (and map_is_valid FALSE) until + // load_refcounts / recovery_begin populates the map, so a premature use of + // the allocator faults at once. return STATUS_OK; } diff --git a/src/superblock.c b/src/superblock.c index 2f35709b..d7d85168 100644 --- a/src/superblock.c +++ b/src/superblock.c @@ -181,9 +181,9 @@ superblock_format(superblock_context *ctx, const allocator_config *cfg) ctx->image->format_version = SUPERBLOCK_FORMAT_VERSION; ctx->image->generation = 0; ctx->image->allocation_state_addr = 0; // fresh DB: rebuild on crash - for (uint64 i = 0; i < SUPERBLOCK_MAX_TREES; i++) { - ctx->image->trees[i].table_id = INVALID_ALLOCATOR_ROOT_ID; - } + // A fresh, empty tree: no root and nothing incorporated yet. + ctx->image->tree.incorporated_generation = + SUPERBLOCK_NO_INCORPORATED_GENERATION; /* * Write both physical copies so torn-write protection is in force from the @@ -217,67 +217,16 @@ superblock_set_allocation_state_addr(superblock_context *ctx, uint64 addr) ctx->image->allocation_state_addr = addr; } -platform_status +void superblock_get_tree_record(const superblock_context *ctx, - allocator_root_id table_id, superblock_tree_record *out) { - platform_assert(table_id != INVALID_ALLOCATOR_ROOT_ID); - for (uint64 i = 0; i < SUPERBLOCK_MAX_TREES; i++) { - if (ctx->image->trees[i].table_id == table_id) { - *out = ctx->image->trees[i]; - return STATUS_OK; - } - } - return STATUS_NOT_FOUND; + *out = ctx->image->tree; } -platform_status +void superblock_set_tree_record(superblock_context *ctx, const superblock_tree_record *rec) { - platform_assert(rec->table_id != INVALID_ALLOCATOR_ROOT_ID); - - // Overwrite an existing record for this table_id if present. - for (uint64 i = 0; i < SUPERBLOCK_MAX_TREES; i++) { - if (ctx->image->trees[i].table_id == rec->table_id) { - ctx->image->trees[i] = *rec; - return STATUS_OK; - } - } - // Otherwise claim the first free slot. - for (uint64 i = 0; i < SUPERBLOCK_MAX_TREES; i++) { - if (ctx->image->trees[i].table_id == INVALID_ALLOCATOR_ROOT_ID) { - ctx->image->trees[i] = *rec; - return STATUS_OK; - } - } - return STATUS_NO_SPACE; -} - -platform_status -superblock_remove_tree_record(superblock_context *ctx, - allocator_root_id table_id) -{ - platform_assert(table_id != INVALID_ALLOCATOR_ROOT_ID); - for (uint64 i = 0; i < SUPERBLOCK_MAX_TREES; i++) { - if (ctx->image->trees[i].table_id == table_id) { - ZERO_CONTENTS(&ctx->image->trees[i]); - ctx->image->trees[i].table_id = INVALID_ALLOCATOR_ROOT_ID; - return STATUS_OK; - } - } - return STATUS_NOT_FOUND; -} - -uint64 -superblock_num_trees(const superblock_context *ctx) -{ - uint64 count = 0; - for (uint64 i = 0; i < SUPERBLOCK_MAX_TREES; i++) { - if (ctx->image->trees[i].table_id != INVALID_ALLOCATOR_ROOT_ID) { - count++; - } - } - return count; + ctx->image->tree = *rec; } diff --git a/src/superblock.h b/src/superblock.h index 487eae46..a668a3d6 100644 --- a/src/superblock.h +++ b/src/superblock.h @@ -24,8 +24,10 @@ * (recovered from the other page), never the geometry the bootstrap read * depends on. * - * This is intentionally a new on-disk format; older databases must be - * reformatted. + * The instance holds a single tree today: its per-tree record is embedded + * directly rather than as an array. Supporting multiple trees would be an + * on-disk format change (version bump + reformat), not just a machinery + * change. */ #pragma once @@ -36,13 +38,6 @@ #include "platform_io.h" #include "util.h" -/* - * Format headroom for multiple trees. The current system asserts at most one - * occupied record (see the caller's claim path); the array is sized so that - * multi-tree support becomes a machinery change, not a format change. - */ -#define SUPERBLOCK_MAX_TREES (30) - #define SUPERBLOCK_FORMAT_MAGIC (0x5344425355504552ULL) // SDBSUPER #define SUPERBLOCK_FORMAT_VERSION (1) @@ -50,15 +45,17 @@ #define SUPERBLOCK_NUM_SLOTS (2) /* - * A per-tree durable record. table_id == INVALID_ALLOCATOR_ROOT_ID marks an - * empty slot. log_meta_head is format headroom for the per-tree segmented log - * (the oldest live segment, from whose start recovery replays); it is 0 until - * the log is wired. No intra-segment replay offset is stored: recovery replays + * The durable per-tree record. The instance always has exactly one tree (from + * mkfs onward), so the record carries no id or occupancy marker. log_meta_head + * is format headroom for the segmented log (the oldest live segment, from whose + * start recovery replays); it is 0 until the log is wired. + * + * There is intentionally no clean/dirty ("unmounted") flag: recovery replays * from a segment boundary and skips records whose generation is at or below - * incorporated_generation. + * incorporated_generation, which makes a clean root's replay a no-op. So the + * superblock's allocation_state_addr validity is the single at-rest signal. */ typedef struct ONDISK superblock_tree_record { - uint64 table_id; uint64 root_addr; uint64 log_meta_head; // oldest live log segment = replay start (0 until wired) /* @@ -68,8 +65,6 @@ typedef struct ONDISK superblock_tree_record { * the log is wired, the replay-skip boundary. */ uint64 incorporated_generation; - bool32 unmounted; // TRUE iff root_addr is a clean-unmount root - uint32 pad; // explicit: keep trailing on-disk bytes deterministic } superblock_tree_record; /* Sentinel for superblock_tree_record.incorporated_generation. */ @@ -87,7 +82,7 @@ typedef struct ONDISK superblock { * root; a clean unmount writes it nonzero after the map is durable. */ uint64 allocation_state_addr; - superblock_tree_record trees[SUPERBLOCK_MAX_TREES]; + superblock_tree_record tree; checksum128 checksum; } superblock; @@ -106,8 +101,8 @@ typedef struct superblock_context { platform_heap_id heap_id; uint64 page_size; buffer_handle image_buffer; - superblock *image; // page-aligned, page-sized - uint64 current_slot; // slot the in-memory image was last read/written + superblock *image; // page-aligned, page-sized + uint64 current_slot; // slot the in-memory image was last read/written } superblock_context; /* @@ -120,10 +115,10 @@ superblock_read_geometry(const char *filename, disk_geometry *geometry); /* Allocate the in-memory image. Does no I/O. */ platform_status -superblock_context_init(superblock_context *ctx, - io_handle *io, +superblock_context_init(superblock_context *ctx, + io_handle *io, const allocator_config *cfg, - platform_heap_id hid); + platform_heap_id hid); void superblock_context_deinit(superblock_context *ctx); @@ -165,33 +160,15 @@ superblock_allocation_state_addr(const superblock_context *ctx); void superblock_set_allocation_state_addr(superblock_context *ctx, uint64 addr); -/* - * Copy the tree record for table_id into *out. Returns STATUS_NOT_FOUND if - * table_id has no record. - */ -platform_status +/* Copy the (always-present) tree record into *out. */ +void superblock_get_tree_record(const superblock_context *ctx, - allocator_root_id table_id, superblock_tree_record *out); /* - * Upsert rec into the in-memory image, matched by rec->table_id. A new - * table_id claims a free slot; STATUS_NO_SPACE if none remain. In-memory - * only; not durable until superblock_publish(). + * Store rec as the tree record. In-memory only; not durable until + * superblock_publish(). */ -platform_status +void superblock_set_tree_record(superblock_context *ctx, const superblock_tree_record *rec); - -/* - * Remove the tree record for table_id from the in-memory image, freeing its - * slot. Returns STATUS_NOT_FOUND if table_id has no record. In-memory only; - * not durable until superblock_publish(). - */ -platform_status -superblock_remove_tree_record(superblock_context *ctx, - allocator_root_id table_id); - -/* Number of occupied tree records in the in-memory image. */ -uint64 -superblock_num_trees(const superblock_context *ctx); diff --git a/tests/unit/superblock_test.c b/tests/unit/superblock_test.c index 8bdeb4ff..b6989085 100644 --- a/tests/unit/superblock_test.c +++ b/tests/unit/superblock_test.c @@ -93,15 +93,20 @@ CTEST2(superblock, test_format_sets_fresh_state) rc = superblock_format(&ctx, &data->allocator_cfg); ASSERT_TRUE(SUCCESS(rc)); ASSERT_FALSE(superblock_allocation_state_valid(&ctx)); - ASSERT_EQUAL(0, superblock_num_trees(&ctx)); + superblock_tree_record rec; + superblock_get_tree_record(&ctx, &rec); + ASSERT_EQUAL(0, rec.root_addr); // empty tree + ASSERT_EQUAL(SUPERBLOCK_NO_INCORPORATED_GENERATION, + rec.incorporated_generation); superblock_context_deinit(&ctx); rc = superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); ASSERT_TRUE(SUCCESS(rc)); rc = superblock_mount(&ctx, &data->allocator_cfg); ASSERT_TRUE(SUCCESS(rc)); - ASSERT_EQUAL(0, superblock_num_trees(&ctx)); ASSERT_FALSE(superblock_allocation_state_valid(&ctx)); + superblock_get_tree_record(&ctx, &rec); + ASSERT_EQUAL(0, rec.root_addr); superblock_context_deinit(&ctx); } @@ -119,14 +124,11 @@ CTEST2(superblock, test_publish_persists_tree_record) ASSERT_TRUE(SUCCESS(rc)); superblock_tree_record rec = { - .table_id = 1, .root_addr = 0x4000, .log_meta_head = 0, .incorporated_generation = SUPERBLOCK_NO_INCORPORATED_GENERATION, - .unmounted = TRUE, }; - rc = superblock_set_tree_record(&ctx, &rec); - ASSERT_TRUE(SUCCESS(rc)); + superblock_set_tree_record(&ctx, &rec); superblock_set_allocation_state_addr(&ctx, 0x8000); rc = superblock_publish(&ctx); ASSERT_TRUE(SUCCESS(rc)); @@ -136,15 +138,12 @@ CTEST2(superblock, test_publish_persists_tree_record) ASSERT_TRUE(SUCCESS(rc)); rc = superblock_mount(&ctx, &data->allocator_cfg); ASSERT_TRUE(SUCCESS(rc)); - ASSERT_EQUAL(1, superblock_num_trees(&ctx)); ASSERT_TRUE(superblock_allocation_state_valid(&ctx)); ASSERT_EQUAL(0x8000, superblock_allocation_state_addr(&ctx)); superblock_tree_record got; - rc = superblock_get_tree_record(&ctx, 1, &got); - ASSERT_TRUE(SUCCESS(rc)); + superblock_get_tree_record(&ctx, &got); ASSERT_EQUAL(0x4000, got.root_addr); - ASSERT_TRUE(got.unmounted); superblock_context_deinit(&ctx); } @@ -165,13 +164,10 @@ CTEST2(superblock, test_torn_write_falls_back_to_older_generation) // After format the image is gen 2 in slot 1, so this publish targets slot 0. superblock_tree_record rec = { - .table_id = 1, .root_addr = 0x4000, .incorporated_generation = SUPERBLOCK_NO_INCORPORATED_GENERATION, - .unmounted = TRUE, }; - rc = superblock_set_tree_record(&ctx, &rec); - ASSERT_TRUE(SUCCESS(rc)); + superblock_set_tree_record(&ctx, &rec); rc = superblock_publish(&ctx); ASSERT_TRUE(SUCCESS(rc)); superblock_context_deinit(&ctx); @@ -179,17 +175,17 @@ CTEST2(superblock, test_torn_write_falls_back_to_older_generation) // Simulate a torn write of the newest slot (slot 0, gen 3). superblock_test_corrupt_slot(data->ioh, data->io_cfg.page_size, 0); - // Mount must still succeed, falling back to slot 1 (gen 2) -- which has no - // tree record, proving the older generation was left intact. + // Mount must still succeed, falling back to slot 1 (gen 2), which carries the + // format's empty tree (root_addr 0), not the published root -- proving the + // older generation was left intact by the torn write. rc = superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); ASSERT_TRUE(SUCCESS(rc)); rc = superblock_mount(&ctx, &data->allocator_cfg); ASSERT_TRUE(SUCCESS(rc)); - ASSERT_EQUAL(0, superblock_num_trees(&ctx)); superblock_tree_record got; - rc = superblock_get_tree_record(&ctx, 1, &got); - ASSERT_FALSE(SUCCESS(rc)); + superblock_get_tree_record(&ctx, &got); + ASSERT_EQUAL(0, got.root_addr); superblock_context_deinit(&ctx); } From 44e10194f191e3c5c99d3bd51600dd3a2e504642 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sat, 25 Jul 2026 23:51:36 -0700 Subject: [PATCH 21/64] cleaning up log Signed-off-by: Rob Johnson --- src/core.c | 12 ++- src/log.h | 81 +++++------------ src/shard_log.c | 171 +++++++----------------------------- src/shard_log.h | 20 ++--- tests/functional/log_test.c | 152 +++++++++++++++----------------- 5 files changed, 143 insertions(+), 293 deletions(-) diff --git a/src/core.c b/src/core.c index 2282bad9..9d3e5117 100644 --- a/src/core.c +++ b/src/core.c @@ -2276,10 +2276,16 @@ core_teardown_after_shutdown(core_handle *spl) // Keep this after checkpoint publication: it supplies the generation cut. memtable_context_deinit(&spl->mt_ctxt); - // Keep the log alive through clean-record publication. A later explicit - // tail-sync protocol will own its immutable log metadata separately. + /* + * The log is not yet wired to the superblock, so we simply seal it -- which + * finalizes its pages, releases its in-memory resources, and frees the + * handle. When the log is wired, the live log's identity (captured at + * creation via log_get_segment_info()) will already be recorded in the + * superblock, and unmount will free its extents (log_dec_ref) once the + * segment is no longer needed. + */ if (spl->cfg.use_log) { - platform_free(spl->heap_id, spl->log); + log_seal(spl->log); spl->log = NULL; } diff --git a/src/log.h b/src/log.h index 30366edc..5cba7244 100644 --- a/src/log.h +++ b/src/log.h @@ -34,41 +34,29 @@ typedef int (*log_write_fn)(log_handle *log, uint64 memtable_generation, uint64 leaf_generation); /* - * Finalize the current append pages into checksummed, immutable pages. + * Finalize and retire the log stream, terminally. Finalizes the current + * append pages into checksummed, immutable pages, releases in-memory + * resources, and frees the handle (which is invalid afterward). * - * The caller must exclude concurrent log_write() calls until it has taken the - * cache writeback fence that is to make the pages durable. It must also - * serialize concurrent log_seal() calls. seal() itself does not issue I/O or - * a durable barrier. - * - * This is deliberately not a persisted replay-boundary descriptor. A log - * implementation whose metadata can grow after sealing needs an additional - * boundary in the checkpoint record to exclude those later entries. + * The caller must exclude concurrent log_write() and log_seal() calls. seal() + * itself issues no writeback or durable barrier: to make the sealed pages + * durable, the caller takes the cache writeback fence + a durable barrier + * afterward. The stream's identity is fixed at log_create() and obtained then + * via log_get_segment_info(), so seal needs no identity out-parameter; the + * caller frees the on-disk extents later via log_dec_ref(). */ typedef platform_status (*log_seal_fn)(log_handle *log); /* - * Detach the current stream after sealing it and prepare a distinct fresh - * stream. The caller must exclude writes throughout the operation and make - * the returned identities durable before allowing writes to the fresh stream. - * A zero sealed.meta_addr means the old stream was empty and discarded. - * Rotation alone does not advance the logical durable-log tail: that requires - * a separate, durable tail/manifest publication by the caller. + * The stream's durable identity, fixed at creation. The caller records it + * (e.g. in the superblock) as soon as the log is created, so that a crash + * mid-stream can find the stream for replay. */ -typedef platform_status (*log_rotate_fn)(log_handle *log, - log_segment_info *sealed, - log_segment_info *fresh); -typedef void (*log_release_fn)(log_handle *log); -typedef uint64 (*log_addr_fn)(log_handle *log); -typedef uint64 (*log_magic_fn)(log_handle *log); +typedef log_segment_info (*log_segment_info_fn)(log_handle *log); typedef struct log_ops { - log_write_fn write; - log_seal_fn seal; - log_rotate_fn rotate; - log_release_fn release; - log_addr_fn addr; - log_addr_fn meta_addr; - log_magic_fn magic; + log_write_fn write; + log_seal_fn seal; + log_segment_info_fn segment_info; } log_ops; // to sub-class log, make a log_handle your first field @@ -88,8 +76,10 @@ log_write(log_handle *log, } /* - * Finalize the log's current append pages. See log_seal_fn for the required - * exclusion, boundary, and durability ordering. + * Finalize and retire the log, freeing the handle. See log_seal_fn for the + * required exclusion and durability ordering; the handle is invalid after this + * returns. Capture the identity via log_get_segment_info() beforehand (it is + * fixed at creation). */ static inline platform_status log_seal(log_handle *log) @@ -97,34 +87,11 @@ log_seal(log_handle *log) return log->ops->seal(log); } -static inline platform_status -log_rotate(log_handle *log, log_segment_info *sealed, log_segment_info *fresh) -{ - return log->ops->rotate(log, sealed, fresh); -} - -static inline void -log_release(log_handle *log) -{ - log->ops->release(log); -} - -static inline uint64 -log_addr(log_handle *log) -{ - return log->ops->addr(log); -} - -static inline uint64 -log_meta_addr(log_handle *log) -{ - return log->ops->meta_addr(log); -} - -static inline uint64 -log_magic(log_handle *log) +/* The stream's durable identity (fixed at creation). See log_segment_info_fn. */ +static inline log_segment_info +log_get_segment_info(log_handle *log) { - return log->ops->magic(log); + return log->ops->segment_info(log); } log_handle * diff --git a/src/shard_log.c b/src/shard_log.c index 1db98edd..862a4465 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -33,27 +33,13 @@ shard_log_write(log_handle *log, uint64 leaf_generation); platform_status shard_log_seal(log_handle *log); -platform_status -shard_log_rotate(log_handle *log, - log_segment_info *sealed, - log_segment_info *fresh); -void -shard_log_release(log_handle *log); -uint64 -shard_log_addr(log_handle *log); -uint64 -shard_log_meta_addr(log_handle *log); -uint64 -shard_log_magic(log_handle *log); +log_segment_info +shard_log_get_segment_info(log_handle *log); static log_ops shard_log_ops = { - .write = shard_log_write, - .seal = shard_log_seal, - .rotate = shard_log_rotate, - .release = shard_log_release, - .addr = shard_log_addr, - .meta_addr = shard_log_meta_addr, - .magic = shard_log_magic, + .write = shard_log_write, + .seal = shard_log_seal, + .segment_info = shard_log_get_segment_info, }; void @@ -114,7 +100,7 @@ shard_log_alloc(shard_log *log, uint64 *next_extent) return cache_alloc(log->cc, addr, PAGE_TYPE_LOG); } -platform_status +static platform_status shard_log_init(shard_log *log, cache *cc, shard_log_config *cfg) { memset(log, 0, sizeof(shard_log)); @@ -149,28 +135,6 @@ shard_log_init(shard_log *log, cache *cc, shard_log_config *cfg) return STATUS_OK; } -void -shard_log_zap(shard_log *log) -{ - cache *cc = log->cc; - - for (threadid i = 0; i < MAX_THREADS; i++) { - shard_log_thread_data *thread_data = shard_log_get_thread_data(log, i); - thread_data->addr = SHARD_UNMAPPED; - thread_data->offset = 0; - } - - if (log->meta_head != 0) { - /* Drop unused per-batch reserves before releasing the mini root. */ - mini_release(&log->mini); - refcount ref = mini_dec_ref(cc, log->meta_head, PAGE_TYPE_LOG); - platform_assert(ref == 0); - log->meta_head = 0; - log->addr = 0; - log->has_pages = FALSE; - } -} - /* * ------------------------------------------------------------------------- * Header for a key/message pair stored in the sharded log: Disk-resident @@ -385,18 +349,18 @@ shard_log_write(log_handle *logh, /* * shard_log_seal -- * - * Finalize every currently active per-thread append page. This is - * deliberately bounded by MAX_THREADS: it does not walk the historical - * log. A final terminal record (where there is room) and checksum make - * each page readable by shard_log_iterator_init(), then clearing the - * append cursor ensures a subsequent writer allocates a new page instead - * of changing the sealed one. + * Finalize and retire a log stream, terminally. Finalizes every currently + * active per-thread append page (bounded by MAX_THREADS; it does not walk + * the historical log): a terminal record (where there is room) and checksum + * make each page readable by shard_log_iterator_init(). Then it releases + * the mini-allocator's unused reserve and frees the handle. After seal the + * handle is invalid; the caller retains the identity it captured earlier + * (shard_log_get_segment_info(), fixed at creation) to reopen the stream + * for replay and, eventually, to free its extents via shard_log_dec_ref(). * * The caller must prevent concurrent shard_log_write() and seal calls. - * In particular, thread_data is otherwise only protected by the - * per-thread writer convention, not by a log-wide lock. This function - * intentionally does not issue writeback: after establishing that - * exclusion, the caller takes cache_writeback_dirty(), followed by a + * seal itself issues no writeback or durable barrier: to make the sealed + * pages durable, the caller takes cache_writeback_dirty() followed by a * durable barrier. */ platform_status @@ -444,73 +408,19 @@ shard_log_seal(log_handle *logh) thread_data->offset = 0; } - return STATUS_OK; -} - -/* - * Detach a sealed stream from the live log. The original allocation reference - * on its metadata extent is transferred to sealed; an eventual durable - * descriptor must own that reference. A fresh mini allocator is prepared - * before the old stream is sealed, so core can publish the fresh identity - * while inserts remain excluded. This only prepares a physical boundary; it - * does not publish or advance the logical durable-log tail. - */ -platform_status -shard_log_rotate(log_handle *logh, - log_segment_info *sealed, - log_segment_info *fresh_info) -{ - shard_log *log = (shard_log *)logh; - shard_log fresh; - - platform_assert(sealed != NULL); - platform_assert(fresh_info != NULL); - ZERO_CONTENTS(sealed); - ZERO_CONTENTS(fresh_info); - - platform_status rc = shard_log_init(&fresh, log->cc, log->cfg); - if (!SUCCESS(rc)) { - return rc; - } - - rc = shard_log_seal(logh); - if (!SUCCESS(rc)) { - shard_log_zap(&fresh); - return rc; - } - - /* No future allocation may use the old mini allocator. */ + /* + * The stream is now immutable. Release the mini-allocator's unused + * per-batch reserve so no future allocation touches this stream, and free + * the handle. The caller already holds the stream's identity (captured at + * creation) and later frees the on-disk extents via shard_log_dec_ref(). + */ mini_release(&log->mini); - - if (log->has_pages) { - *sealed = (log_segment_info){ - .addr = log->addr, - .meta_addr = log->meta_head, - .magic = log->magic, - }; - } else { - /* An empty segment has no descriptor and no retained ownership. */ - refcount ref = mini_dec_ref(log->cc, log->meta_head, PAGE_TYPE_LOG); - platform_assert(ref == 0); - } - - log->mini = fresh.mini; - log->addr = fresh.addr; - log->meta_head = fresh.meta_head; - log->magic = fresh.magic; - log->has_pages = FALSE; - memcpy(log->thread_data, fresh.thread_data, sizeof(log->thread_data)); - - *fresh_info = (log_segment_info){ - .addr = log->addr, - .meta_addr = log->meta_head, - .magic = log->magic, - }; + platform_free(log->heap_id, log); return STATUS_OK; } void -shard_log_segment_discard(cache *cc, const log_segment_info *segment) +shard_log_dec_ref(cache *cc, const log_segment_info *segment) { if (segment->meta_addr == 0) { return; @@ -519,31 +429,15 @@ shard_log_segment_discard(cache *cc, const log_segment_info *segment) platform_assert(ref == 0); } -void -shard_log_release(log_handle *logh) -{ - shard_log_zap((shard_log *)logh); -} - -uint64 -shard_log_addr(log_handle *logh) -{ - shard_log *log = (shard_log *)logh; - return log->addr; -} - -uint64 -shard_log_meta_addr(log_handle *logh) +log_segment_info +shard_log_get_segment_info(log_handle *logh) { shard_log *log = (shard_log *)logh; - return log->meta_head; -} - -uint64 -shard_log_magic(log_handle *logh) -{ - shard_log *log = (shard_log *)logh; - return log->magic; + return (log_segment_info){ + .addr = log->addr, + .meta_addr = log->meta_head, + .magic = log->magic, + }; } bool32 @@ -599,6 +493,9 @@ log_create(cache *cc, log_config *lcfg, platform_heap_id hid) platform_free(hid, slog); return NULL; } + // Remember the heap so log_seal() can free the handle without the caller + // touching platform_free() directly. + slog->heap_id = hid; return (log_handle *)slog; } diff --git a/src/shard_log.h b/src/shard_log.h index 086381a7..32c1d171 100644 --- a/src/shard_log.h +++ b/src/shard_log.h @@ -41,6 +41,7 @@ typedef struct shard_log { log_handle super; // handle to log I/O ops abstraction. cache *cc; shard_log_config *cfg; + platform_heap_id heap_id; // heap the handle was allocated from; freed by seal shard_log_thread_data thread_data[MAX_THREADS]; mini_allocator mini; uint64 addr; @@ -75,24 +76,13 @@ typedef struct ONDISK shard_log_hdr { uint16 num_entries; } shard_log_hdr; -platform_status -shard_log_init(shard_log *log, cache *cc, shard_log_config *cfg); - -platform_status -shard_log_rotate(log_handle *log, - log_segment_info *sealed, - log_segment_info *fresh); - -void -shard_log_zap(shard_log *log); - /* - * Drop the external mini-allocator ownership transferred to a detached - * segment. It is for failed rotation cleanup and tests; persisted segment - * descriptors will eventually own and release this reference instead. + * Release a sealed log segment: drop the mini-allocator reference its metadata + * extent holds, freeing the segment's on-disk extents. Identified by + * log_segment_info (no handle needed -- the handle was freed by log_seal()). */ void -shard_log_segment_discard(cache *cc, const log_segment_info *segment); +shard_log_dec_ref(cache *cc, const log_segment_info *segment); platform_status shard_log_iterator_init(cache *cc, diff --git a/tests/functional/log_test.c b/tests/functional/log_test.c index fa17d8b4..f1c4ec6c 100644 --- a/tests/functional/log_test.c +++ b/tests/functional/log_test.c @@ -27,7 +27,6 @@ test_log_crash(clockcache *cc, io_handle *io, allocator *al, shard_log_config *cfg, - shard_log *log, task_system *ts, platform_heap_id hid, test_message_generator *gen, @@ -41,8 +40,7 @@ test_log_crash(clockcache *cc, uint64 i; key returned_key; message returned_message; - uint64 addr; - uint64 magic; + log_segment_info segment; shard_log_iterator itor; iterator *itorh = (iterator *)&itor; char key_str[128]; @@ -51,12 +49,11 @@ test_log_crash(clockcache *cc, DECLARE_AUTO_KEY_BUFFER(keybuffer, hid); platform_assert(cc != NULL); - rc = shard_log_init(log, (cache *)cc, cfg); - platform_assert_status_ok(rc); - logh = (log_handle *)log; + logh = log_create((cache *)cc, (log_config *)cfg, hid); + platform_assert(logh != NULL); - addr = log_addr(logh); - magic = log_magic(logh); + // The identity is fixed at creation; capture it before writing/sealing. + segment = log_get_segment_info(logh); merge_accumulator_init(&msg, hid); @@ -92,7 +89,7 @@ test_log_crash(clockcache *cc, platform_assert(log_rc == 0); } - rc = log_seal(logh); + rc = log_seal(logh); // frees logh; identity captured above platform_assert_status_ok(rc); rc = cache_writeback_dirty((cache *)cc); platform_assert_status_ok(rc); @@ -106,7 +103,8 @@ test_log_crash(clockcache *cc, platform_assert_status_ok(rc); } - rc = shard_log_iterator_init((cache *)cc, cfg, hid, addr, magic, &itor); + rc = shard_log_iterator_init( + (cache *)cc, cfg, hid, segment.addr, segment.magic, &itor); platform_assert_status_ok(rc); itorh = (iterator *)&itor; @@ -145,7 +143,7 @@ test_log_crash(clockcache *cc, merge_accumulator_deinit(&msg); shard_log_iterator_deinit(hid, &itor); - shard_log_zap(log); + shard_log_dec_ref((cache *)cc, &segment); return 0; } @@ -236,39 +234,34 @@ test_log_verify_segment(cache *cc, } /* - * A rotation must permanently detach the first mini-allocator stream and - * produce a fresh one. Reinitializing the cache after each forced physical - * persistence cut makes this test exercise only persisted pages for both - * identities; it does not model logical durable-tail publication. + * Sealing a stream and creating a fresh one must yield two distinct, + * independently replayable segments. Reinitializing the cache after each forced + * physical persistence cut makes this test exercise only persisted pages for + * both identities; it does not model logical durable-tail publication. */ static int -test_log_rotate(clockcache *cc, - clockcache_config *cache_cfg, - io_handle *io, - allocator *al, - shard_log_config *cfg, - shard_log *log, - platform_heap_id hid, - test_message_generator *gen, - uint64 key_size) +test_log_two_segments(clockcache *cc, + clockcache_config *cache_cfg, + io_handle *io, + allocator *al, + shard_log_config *cfg, + platform_heap_id hid, + test_message_generator *gen, + uint64 key_size) { const uint64 old_first = 1000, old_count = 16; const uint64 new_first = 2000, new_count = 16; log_segment_info sealed, fresh; - platform_status rc = shard_log_init(log, (cache *)cc, cfg); - platform_assert_status_ok(rc); + log_handle *log = log_create((cache *)cc, (log_config *)cfg, hid); + platform_assert(log != NULL); + sealed = log_get_segment_info(log); // identity is fixed at creation + test_log_write_range(log, gen, hid, key_size, old_first, old_count); - test_log_write_range( - (log_handle *)log, gen, hid, key_size, old_first, old_count); - rc = log_rotate((log_handle *)log, &sealed, &fresh); + platform_status rc = log_seal(log); // frees log platform_assert_status_ok(rc); platform_assert(sealed.addr != 0); platform_assert(sealed.meta_addr != 0); - platform_assert(fresh.addr != 0); - platform_assert(fresh.meta_addr != 0); - platform_assert(sealed.meta_addr != fresh.meta_addr); - platform_assert(sealed.magic != fresh.magic); rc = cache_writeback_dirty((cache *)cc); platform_assert_status_ok(rc); @@ -277,15 +270,23 @@ test_log_rotate(clockcache *cc, clockcache_deinit(cc); rc = clockcache_init( - cc, cache_cfg, io, al, "rotated-old", hid, platform_get_module_id()); + cc, cache_cfg, io, al, "sealed-old", hid, platform_get_module_id()); platform_assert_status_ok(rc); test_log_verify_segment( (cache *)cc, cfg, &sealed, gen, hid, key_size, old_first, old_count); - test_log_write_range( - (log_handle *)log, gen, hid, key_size, new_first, new_count); - rc = log_seal((log_handle *)log); + // A fresh stream is a distinct segment: new mini allocator and new magic. + log = log_create((cache *)cc, (log_config *)cfg, hid); + platform_assert(log != NULL); + fresh = log_get_segment_info(log); + test_log_write_range(log, gen, hid, key_size, new_first, new_count); + rc = log_seal(log); // frees log platform_assert_status_ok(rc); + platform_assert(fresh.addr != 0); + platform_assert(fresh.meta_addr != 0); + platform_assert(sealed.meta_addr != fresh.meta_addr); + platform_assert(sealed.magic != fresh.magic); + rc = cache_writeback_dirty((cache *)cc); platform_assert_status_ok(rc); rc = cache_durable_barrier((cache *)cc); @@ -293,25 +294,25 @@ test_log_rotate(clockcache *cc, clockcache_deinit(cc); rc = clockcache_init( - cc, cache_cfg, io, al, "rotated-new", hid, platform_get_module_id()); + cc, cache_cfg, io, al, "sealed-new", hid, platform_get_module_id()); platform_assert_status_ok(rc); test_log_verify_segment( (cache *)cc, cfg, &sealed, gen, hid, key_size, old_first, old_count); test_log_verify_segment( (cache *)cc, cfg, &fresh, gen, hid, key_size, new_first, new_count); - shard_log_segment_discard((cache *)cc, &sealed); - shard_log_zap(log); + shard_log_dec_ref((cache *)cc, &sealed); + shard_log_dec_ref((cache *)cc, &fresh); return 0; } static int test_log_large_message(cache *cc, shard_log_config *cfg, - shard_log *log, platform_heap_id hid) { platform_status rc; + log_segment_info sealed; shard_log_iterator itor; iterator *itorh = (iterator *)&itor; merge_accumulator msg; @@ -321,8 +322,9 @@ test_log_large_message(cache *cc, key skey = key_create(FALSE, sizeof(key_data) - 1, key_data); uint64 value_len = 3 * cache_page_size(cc) + 123; - rc = shard_log_init(log, cc, cfg); - platform_assert_status_ok(rc); + log_handle *logh = log_create(cc, (log_config *)cfg, hid); + platform_assert(logh != NULL); + sealed = log_get_segment_info(logh); // identity is fixed at creation merge_accumulator_init(&msg, hid); bool32 success = merge_accumulator_resize(&msg, value_len); @@ -330,8 +332,8 @@ test_log_large_message(cache *cc, merge_accumulator_set_class(&msg, MESSAGE_TYPE_INSERT); memset(merge_accumulator_data(&msg), 'L', value_len); - int log_rc = log_write( - (log_handle *)log, skey, merge_accumulator_to_message(&msg), 0, 0); + int log_rc = + log_write(logh, skey, merge_accumulator_to_message(&msg), 0, 0); platform_assert(log_rc == 0); merge_accumulator filler; @@ -342,28 +344,21 @@ test_log_large_message(cache *cc, memset( merge_accumulator_data(&filler), 'f', merge_accumulator_length(&filler)); for (uint64 i = 1; i < 16; i++) { - log_rc = log_write((log_handle *)log, - skey, - merge_accumulator_to_message(&filler), - i / 4, - i % 4); + log_rc = log_write( + logh, skey, merge_accumulator_to_message(&filler), i / 4, i % 4); platform_assert(log_rc == 0); } merge_accumulator_deinit(&filler); - rc = log_seal((log_handle *)log); + rc = log_seal(logh); // frees logh; identity captured above platform_assert_status_ok(rc); rc = cache_writeback_dirty(cc); platform_assert_status_ok(rc); rc = cache_durable_barrier(cc); platform_assert_status_ok(rc); - rc = shard_log_iterator_init(cc, - cfg, - hid, - log_addr((log_handle *)log), - log_magic((log_handle *)log), - &itor); + rc = shard_log_iterator_init( + cc, cfg, hid, sealed.addr, sealed.magic, &itor); platform_assert_status_ok(rc); platform_assert(iterator_can_curr(itorh)); @@ -375,12 +370,12 @@ test_log_large_message(cache *cc, shard_log_iterator_deinit(hid, &itor); merge_accumulator_deinit(&msg); - shard_log_zap(log); + shard_log_dec_ref(cc, &sealed); return 0; } typedef struct test_log_thread_params { - shard_log *log; + log_handle *logh; platform_thread thread; int thread_id; test_message_generator *gen; @@ -393,8 +388,7 @@ test_log_thread(void *arg) { platform_heap_id hid = platform_get_heap_id(); test_log_thread_params *params = (test_log_thread_params *)arg; - shard_log *log = params->log; - log_handle *logh = (log_handle *)log; + log_handle *logh = params->logh; int thread_id = params->thread_id; uint64 num_entries = params->num_entries; test_message_generator *gen = params->gen; @@ -419,7 +413,6 @@ test_log_thread(void *arg) platform_status test_log_perf(cache *cc, shard_log_config *cfg, - shard_log *log, uint64 num_entries, test_message_generator *gen, uint64 key_size, @@ -434,11 +427,12 @@ test_log_perf(cache *cc, uint64 start_time; platform_status ret; - ret = shard_log_init(log, (cache *)cc, cfg); - platform_assert_status_ok(ret); + log_handle *logh = log_create((cache *)cc, (log_config *)cfg, hid); + platform_assert(logh != NULL); + log_segment_info sealed = log_get_segment_info(logh); for (uint64 i = 0; i < num_threads; i++) { - params[i].log = log; + params[i].logh = logh; params[i].thread_id = i; params[i].gen = gen; params[i].key_size = key_size; @@ -466,6 +460,9 @@ test_log_perf(cache *cc, / platform_timestamp_elapsed(start_time)); cleanup: + // Seal (frees the handle) and release the segment's extents. + log_seal(logh); + shard_log_dec_ref((cache *)cc, &sealed); platform_free(hid, params); return ret; @@ -584,26 +581,22 @@ log_test(int argc, char *argv[]) platform_get_module_id()); platform_assert_status_ok(status); - shard_log *log = TYPED_MALLOC(hid, log); - platform_assert(log != NULL); - rc = test_log_large_message((cache *)cc, &system_cfg.log_cfg, log, hid); + rc = test_log_large_message((cache *)cc, &system_cfg.log_cfg, hid); platform_assert(rc == 0); - rc = test_log_rotate(cc, - &system_cfg.cache_cfg, - io, - (allocator *)&al, - &system_cfg.log_cfg, - log, - hid, - &gen, - workload_cfg.key_size); + rc = test_log_two_segments(cc, + &system_cfg.cache_cfg, + io, + (allocator *)&al, + &system_cfg.log_cfg, + hid, + &gen, + workload_cfg.key_size); platform_assert(rc == 0); if (run_perf_test) { ret = test_log_perf((cache *)cc, &system_cfg.log_cfg, - log, 200000000, &gen, workload_cfg.key_size, @@ -618,7 +611,6 @@ log_test(int argc, char *argv[]) io, (allocator *)&al, &system_cfg.log_cfg, - log, &ts, hid, &gen, @@ -632,7 +624,6 @@ log_test(int argc, char *argv[]) io, (allocator *)&al, &system_cfg.log_cfg, - log, &ts, hid, &gen, @@ -644,7 +635,6 @@ log_test(int argc, char *argv[]) io_wait_all(io); clockcache_deinit(cc); - platform_free(hid, log); platform_free(hid, cc); rc_allocator_deinit(&al); test_deinit_task_system(&ts); From 82944afb88d3fb6cb82286ef3ff5f2d820bbc7d2 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sun, 26 Jul 2026 00:13:23 -0700 Subject: [PATCH 22/64] two-log foundation Signed-off-by: Rob Johnson --- src/core.c | 16 +++-- src/superblock.h | 43 +++++++++---- tests/unit/superblock_test.c | 120 ++++++++++++++++++++++++++++++++++- 3 files changed, 163 insertions(+), 16 deletions(-) diff --git a/src/core.c b/src/core.c index 9d3e5117..3cf030fd 100644 --- a/src/core.c +++ b/src/core.c @@ -212,10 +212,12 @@ core_publish_root_record(core_handle *spl) ZERO_CONTENTS(&rec); rec.root_addr = snapshot.root_addr; - rec.log_meta_head = 0; // set once the per-tree log is wired rec.incorporated_generation = has_incorporated_generation ? incorporated_generation : SUPERBLOCK_NO_INCORPORATED_GENERATION; + // Carry the log pointers across a root advance (wired fully in Stage B/C). + rec.live_log = old_rec.live_log; + rec.sealed_log = old_rec.sealed_log; superblock_set_tree_record(&spl->superblock, &rec); /* @@ -2439,14 +2441,20 @@ core_print_super_block(platform_log_handle *log_handle, core_handle *spl) platform_log(log_handle, "Superblock tree record root_id=%lu {\n" - " root_addr=%lu log_meta_head=%lu\n" - " incorporated_generation=%lu\n" + " root_addr=%lu incorporated_generation=%lu\n" + " live_log: meta_addr=%lu addr=%lu magic=%lu\n" + " sealed_log: meta_addr=%lu addr=%lu magic=%lu\n" " allocation_state: %s (addr=%lu)\n" "}\n\n", spl->id, rec.root_addr, - rec.log_meta_head, rec.incorporated_generation, + rec.live_log.meta_addr, + rec.live_log.addr, + rec.live_log.magic, + rec.sealed_log.meta_addr, + rec.sealed_log.addr, + rec.sealed_log.magic, superblock_allocation_state_valid(&spl->superblock) ? "valid" : "invalid", superblock_allocation_state_addr(&spl->superblock)); diff --git a/src/superblock.h b/src/superblock.h index a668a3d6..c10cfffc 100644 --- a/src/superblock.h +++ b/src/superblock.h @@ -44,27 +44,48 @@ /* The two physical superblock copies live at pages 0 and 1. */ #define SUPERBLOCK_NUM_SLOTS (2) +/* + * A log's on-disk identity. Mirrors log_segment_info's layout; the superblock + * stores it opaquely and does not depend on the log module. meta_addr == 0 + * means "no log present". + */ +typedef struct ONDISK superblock_log_info { + uint64 addr; + uint64 meta_addr; + uint64 magic; +} superblock_log_info; + +/* An empty (absent) log slot: meta_addr == 0. */ +#define SUPERBLOCK_NO_LOG(info) ((info).meta_addr == 0) + /* * The durable per-tree record. The instance always has exactly one tree (from - * mkfs onward), so the record carries no id or occupancy marker. log_meta_head - * is format headroom for the segmented log (the oldest live segment, from whose - * start recovery replays); it is 0 until the log is wired. + * mkfs onward), so the record carries no id or occupancy marker. + * + * The two log pointers implement the two-log checkpoint protocol: live_log is + * the stream currently receiving inserts; sealed_log is set only while a + * checkpoint is in progress -- the just-sealed stream whose entries are being + * folded into the new root. At rest (between checkpoints, and after a clean + * unmount) sealed_log is empty. On crash recovery the sealed log (if present) + * then the live log are replayed onto root_addr, skipping entries at or below + * incorporated_generation. * - * There is intentionally no clean/dirty ("unmounted") flag: recovery replays - * from a segment boundary and skips records whose generation is at or below - * incorporated_generation, which makes a clean root's replay a no-op. So the - * superblock's allocation_state_addr validity is the single at-rest signal. + * There is intentionally no clean/dirty ("unmounted") flag: replaying a clean + * root is a no-op because every entry is at or below incorporated_generation, + * so the superblock's allocation_state_addr validity is the single at-rest + * signal. */ typedef struct ONDISK superblock_tree_record { uint64 root_addr; - uint64 log_meta_head; // oldest live log segment = replay start (0 until wired) /* * Highest memtable generation folded into root_addr; UINT64_MAX means none * has been incorporated yet (distinct from generation 0, which is a real, - * live generation). Drives memtable-generation resume at mount and, once - * the log is wired, the replay-skip boundary. + * live generation). Drives memtable-generation resume at mount and the + * replay-skip boundary. */ - uint64 incorporated_generation; + uint64 incorporated_generation; + superblock_log_info live_log; + superblock_log_info sealed_log; } superblock_tree_record; /* Sentinel for superblock_tree_record.incorporated_generation. */ diff --git a/tests/unit/superblock_test.c b/tests/unit/superblock_test.c index b6989085..a07008b3 100644 --- a/tests/unit/superblock_test.c +++ b/tests/unit/superblock_test.c @@ -125,8 +125,8 @@ CTEST2(superblock, test_publish_persists_tree_record) superblock_tree_record rec = { .root_addr = 0x4000, - .log_meta_head = 0, .incorporated_generation = SUPERBLOCK_NO_INCORPORATED_GENERATION, + .live_log = {.addr = 0x6000, .meta_addr = 0x8000, .magic = 0x11}, }; superblock_set_tree_record(&ctx, &rec); superblock_set_allocation_state_addr(&ctx, 0x8000); @@ -144,6 +144,124 @@ CTEST2(superblock, test_publish_persists_tree_record) superblock_tree_record got; superblock_get_tree_record(&ctx, &got); ASSERT_EQUAL(0x4000, got.root_addr); + ASSERT_EQUAL(0x6000, got.live_log.addr); + ASSERT_EQUAL(0x8000, got.live_log.meta_addr); + ASSERT_EQUAL(0x11, got.live_log.magic); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(got.sealed_log)); // no checkpoint in progress + superblock_context_deinit(&ctx); +} + +/* Steady, begin-checkpoint, and complete-checkpoint tree-record states. */ +static const superblock_log_info TEST_LOG_L1 = { + .addr = 0x6000, .meta_addr = 0x8000, .magic = 0x11}; +static const superblock_log_info TEST_LOG_L2 = { + .addr = 0x10000, .meta_addr = 0x12000, .magic = 0x22}; + +/* + * Walk the two-log checkpoint state machine through the superblock and confirm + * each published state reads back: steady {root R0, live L1, no sealed} -> + * begin {root R0, sealed L1, live L2} -> complete {root R1, live L2, no sealed}. + */ +CTEST2(superblock, test_two_log_checkpoint_transitions) +{ + superblock_context ctx; + platform_status rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_format(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + + superblock_tree_record rec = { + .root_addr = 0x4000, + .incorporated_generation = 5, + .live_log = TEST_LOG_L1, + }; + superblock_set_tree_record(&ctx, &rec); + rc = superblock_publish(&ctx); + ASSERT_TRUE(SUCCESS(rc)); + + // Begin: seal L1 -> sealed_log, new live L2. + rec.sealed_log = rec.live_log; + rec.live_log = TEST_LOG_L2; + superblock_set_tree_record(&ctx, &rec); + rc = superblock_publish(&ctx); + ASSERT_TRUE(SUCCESS(rc)); + + superblock_tree_record got; + superblock_get_tree_record(&ctx, &got); + ASSERT_EQUAL(TEST_LOG_L1.meta_addr, got.sealed_log.meta_addr); + ASSERT_EQUAL(TEST_LOG_L2.meta_addr, got.live_log.meta_addr); + + // Complete: advance root, clear sealed_log. + rec.root_addr = 0x4400; + rec.incorporated_generation = 9; + rec.sealed_log = (superblock_log_info){0}; + superblock_set_tree_record(&ctx, &rec); + rc = superblock_publish(&ctx); + ASSERT_TRUE(SUCCESS(rc)); + superblock_context_deinit(&ctx); + + // A fresh mount reads the completed state. + rc = superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_mount(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + superblock_get_tree_record(&ctx, &got); + ASSERT_EQUAL(0x4400, got.root_addr); + ASSERT_EQUAL(9, got.incorporated_generation); + ASSERT_EQUAL(TEST_LOG_L2.meta_addr, got.live_log.meta_addr); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(got.sealed_log)); + superblock_context_deinit(&ctx); +} + +/* + * A torn begin-checkpoint publish must leave the previous (steady) generation + * intact: mount falls back to {root R0, live L1, no sealed}, which recovery can + * replay -- never to a half-written checkpoint. + */ +CTEST2(superblock, test_two_log_checkpoint_torn_begin) +{ + superblock_context ctx; + platform_status rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_format(&ctx, &data->allocator_cfg); // slot0=gen1, slot1=gen2 + ASSERT_TRUE(SUCCESS(rc)); + + // Publish the steady state into BOTH slots (gen3->slot0, gen4->slot1), so the + // fallback below is unambiguously the steady state, not the empty format one. + superblock_tree_record rec = { + .root_addr = 0x4000, + .incorporated_generation = 5, + .live_log = TEST_LOG_L1, + }; + superblock_set_tree_record(&ctx, &rec); + rc = superblock_publish(&ctx); // gen3 -> slot0 + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_publish(&ctx); // gen4 -> slot1 + ASSERT_TRUE(SUCCESS(rc)); + + // Begin checkpoint: this publish (gen5) targets slot0. + rec.sealed_log = rec.live_log; + rec.live_log = TEST_LOG_L2; + superblock_set_tree_record(&ctx, &rec); + rc = superblock_publish(&ctx); // gen5 -> slot0 + ASSERT_TRUE(SUCCESS(rc)); + superblock_context_deinit(&ctx); + + // Tear the begin publish (newest slot, slot0/gen5). + superblock_test_corrupt_slot(data->ioh, data->io_cfg.page_size, 0); + + // Mount falls back to slot1 (gen4) = steady: live L1, no sealed. + rc = superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_mount(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + superblock_tree_record got; + superblock_get_tree_record(&ctx, &got); + ASSERT_EQUAL(0x4000, got.root_addr); + ASSERT_EQUAL(TEST_LOG_L1.meta_addr, got.live_log.meta_addr); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(got.sealed_log)); superblock_context_deinit(&ctx); } From bd45a14b3586658b74ec3784279534a54cfcb8a7 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sun, 26 Jul 2026 00:37:31 -0700 Subject: [PATCH 23/64] non-concurrent checkpoint Signed-off-by: Rob Johnson --- src/core.c | 200 ++++++++++++++++++++++++----- src/core.h | 7 + src/log.h | 9 ++ src/shard_log.c | 6 +- src/shard_log.h | 8 -- tests/functional/log_test.c | 10 +- tests/unit/splinter_test.c | 60 +++++++++ tests/unit/splinterdb_quick_test.c | 42 ++++++ 8 files changed, 295 insertions(+), 47 deletions(-) diff --git a/src/core.c b/src/core.c index 3cf030fd..12e9b3be 100644 --- a/src/core.c +++ b/src/core.c @@ -155,17 +155,31 @@ core_capture_checkpoint_cut(core_handle *spl, return STATUS_OK; } +/* + * Translate between the log module's log_segment_info and the superblock's + * (layout-identical but module-independent) superblock_log_info. + */ +static superblock_log_info +core_log_to_superblock(log_segment_info info) +{ + return (superblock_log_info){ + .addr = info.addr, .meta_addr = info.meta_addr, .magic = info.magic}; +} + /* * Publish a tree-record root advance to the superblock: capture the current - * COW root, make it durable, record it (with the incorporated-generation cut), - * invalidate the persisted allocation state in the same atomic update, and - * release the previously published root. Used by mkfs (empty root) and by - * unmount (Part A; unmount then persists the map and republishes a valid - * allocation state). The snapshot's owned reference is transferred to the - * durable record on a successful publish. + * COW root, make it durable, record it (with the incorporated-generation cut + * and the given live/sealed log pointers), invalidate the persisted allocation + * state in the same atomic update, and release the previously published root. + * Used by mkfs (empty root, records the new live log) and by unmount (Part A; + * unmount then persists the map and republishes a valid allocation state). The + * snapshot's owned reference is transferred to the durable record on a + * successful publish. */ static platform_status -core_publish_root_record(core_handle *spl) +core_publish_root_record(core_handle *spl, + superblock_log_info live_log, + superblock_log_info sealed_log) { platform_status rc; trunk_snapshot snapshot; @@ -215,9 +229,8 @@ core_publish_root_record(core_handle *spl) rec.incorporated_generation = has_incorporated_generation ? incorporated_generation : SUPERBLOCK_NO_INCORPORATED_GENERATION; - // Carry the log pointers across a root advance (wired fully in Stage B/C). - rec.live_log = old_rec.live_log; - rec.sealed_log = old_rec.sealed_log; + rec.live_log = live_log; + rec.sealed_log = sealed_log; superblock_set_tree_record(&spl->superblock, &rec); /* @@ -2000,8 +2013,13 @@ core_mkfs(core_handle *spl, goto deinit_trunk_context; } - // Establish the initial (empty) tree record; publish it durably. - rc = core_publish_root_record(spl); + // Establish the initial (empty) tree record, recording the live log; publish + // it durably. No sealed log at mkfs. + superblock_log_info live_log = {0}; + if (spl->cfg.use_log) { + live_log = core_log_to_superblock(log_get_segment_info(spl->log)); + } + rc = core_publish_root_record(spl, live_log, (superblock_log_info){0}); if (!SUCCESS(rc)) { platform_error_log("core_mkfs: core_publish_root_record failed: %s\n", platform_status_to_string(rc)); @@ -2164,12 +2182,18 @@ core_mount(core_handle *spl, } /* - * Mark dirty: invalidate the persisted allocation state before any - * allocation diverges the in-memory map from disk. A crash after this - * forces the next mount into recovery instead of silently reverting to this - * now-stale root. The tree record itself is unchanged (same root), so only - * the allocation state is republished. + * Mark dirty: record this session's fresh live log and invalidate the + * persisted allocation state, before any allocation diverges the in-memory + * map from disk. A crash after this forces the next mount into recovery + * instead of silently reverting to this now-stale root. The root is + * unchanged; a clean mount has no sealed log. */ + superblock_get_tree_record(&spl->superblock, &rec); + rec.sealed_log = (superblock_log_info){0}; + rec.live_log = spl->cfg.use_log + ? core_log_to_superblock(log_get_segment_info(spl->log)) + : (superblock_log_info){0}; + superblock_set_tree_record(&spl->superblock, &rec); superblock_set_allocation_state_addr(&spl->superblock, 0); rc = superblock_publish(&spl->superblock); if (!SUCCESS(rc)) { @@ -2278,21 +2302,119 @@ core_teardown_after_shutdown(core_handle *spl) // Keep this after checkpoint publication: it supplies the generation cut. memtable_context_deinit(&spl->mt_ctxt); - /* - * The log is not yet wired to the superblock, so we simply seal it -- which - * finalizes its pages, releases its in-memory resources, and frees the - * handle. When the log is wired, the live log's identity (captured at - * creation via log_get_segment_info()) will already be recorded in the - * superblock, and unmount will free its extents (log_dec_ref) once the - * segment is no longer needed. - */ + // flush all dirty pages in the cache. The live log has already been sealed + // by the caller (core_seal_live_log); its extents are freed after this flush. + cache_flush(spl->cc); +} + +/* + * Seal the live log at shutdown -- finalizes its pages and frees the handle -- + * and return its identity so the caller can free the extents with log_dec_ref() + * once the cache is flushed. A clean shutdown has folded everything into the + * durable root, so the live log is fully incorporated and discardable. Returns + * an empty descriptor when logging is disabled. + */ +static log_segment_info +core_seal_live_log(core_handle *spl) +{ + if (!spl->cfg.use_log || spl->log == NULL) { + return (log_segment_info){0}; + } + log_segment_info info = log_get_segment_info(spl->log); + log_seal(spl->log); + spl->log = NULL; + return info; +} + +/* + * Take a checkpoint: advance the durable root, rotating the log via the two-log + * protocol. Call at a quiescent point (no concurrent inserts). + * + * Two superblock publishes bracket the incorporation, so a crash mid-checkpoint + * recovers to a well-defined state: + * + * begin -> {root = C_prev, sealed = old live log, live = new log} + * (fold the sealed log's entries into the trunk, advancing the COW root) + * complete -> {root = C_new (incorporates the sealed log), sealed = empty} + * + * then the sealed log's extents are freed. With logging disabled it is simply + * an incorporate + durable-root advance (no log to rotate). + */ +platform_status +core_checkpoint(core_handle *spl) +{ + platform_status rc; + superblock_log_info new_live = {0}; + log_segment_info sealed = {0}; + if (spl->cfg.use_log) { - log_seal(spl->log); + // --- Begin: seal the live log, start a fresh one, and publish + // {sealed = old live, live = new} without advancing the root. --- + rc = platform_mutex_lock(&spl->checkpoint_lock); + if (!SUCCESS(rc)) { + return rc; + } + + sealed = log_get_segment_info(spl->log); + log_seal(spl->log); // finalizes the pages (dirty), frees the handle spl->log = NULL; + + // The sealed log must be fully durable before it is recorded as the + // sealed segment (recovery replays it as-is). Flush the finalized pages. + rc = cache_writeback_dirty(spl->cc); + if (SUCCESS(rc)) { + rc = cache_durable_barrier(spl->cc); + } + if (SUCCESS(rc)) { + spl->log = log_create(spl->cc, spl->cfg.log_cfg, spl->heap_id); + if (spl->log == NULL) { + platform_error_log("core_checkpoint: log_create failed\n"); + rc = STATUS_NO_MEMORY; + } + } + if (SUCCESS(rc)) { + new_live = core_log_to_superblock(log_get_segment_info(spl->log)); + superblock_tree_record rec; + superblock_get_tree_record(&spl->superblock, &rec); + rec.live_log = new_live; + rec.sealed_log = core_log_to_superblock(sealed); + superblock_set_tree_record(&spl->superblock, &rec); + superblock_set_allocation_state_addr(&spl->superblock, 0); + rc = superblock_publish(&spl->superblock); + } + + platform_status unlock_rc = platform_mutex_unlock(&spl->checkpoint_lock); + if (SUCCESS(rc) && !SUCCESS(unlock_rc)) { + rc = unlock_rc; + } + if (!SUCCESS(rc)) { + return rc; + } } - // flush all dirty pages in the cache - cache_flush(spl->cc); + // --- Incorporate: fold everything logged so far into the trunk, advancing + // the in-memory COW root. Synchronous (no concurrent inserts). --- + if (!memtable_is_empty(&spl->mt_ctxt)) { + uint64 generation = memtable_force_finalize(&spl->mt_ctxt); + core_memtable_flush(spl, generation); + } + rc = task_perform_until_quiescent(spl->ts); + if (!SUCCESS(rc)) { + return rc; + } + + // --- Complete: advance the durable root to the incorporated state, keeping + // the new live log and clearing the sealed slot. --- + rc = core_publish_root_record(spl, new_live, (superblock_log_info){0}); + if (!SUCCESS(rc)) { + return rc; + } + + // The sealed log's entries are now durably in the root; free its extents. + if (spl->cfg.use_log) { + log_dec_ref(spl->cc, &sealed); + } + return STATUS_OK; } /* @@ -2312,16 +2434,28 @@ core_unmount(core_handle *spl) core_quiesce_for_shutdown(spl); /* - * Part A: publish the clean-unmount root. Allocation state stays invalid - * here; it becomes valid only in Part B, after the map is persisted. + * The clean-unmount root incorporates everything, so the live log is fully + * folded and discardable. Seal it (frees the handle) now; free its extents + * after the cache flush below. + */ + log_segment_info live_log = core_seal_live_log(spl); + + /* + * Part A: publish the clean-unmount root with both log slots cleared (no + * live or sealed log at rest). Allocation state stays invalid here; it + * becomes valid only in Part B, after the map is persisted. */ - rc = core_publish_root_record(spl); + rc = core_publish_root_record( + spl, (superblock_log_info){0}, (superblock_log_info){0}); if (!SUCCESS(rc)) { platform_error_log("core_unmount: failed to publish unmount root: %s\n", platform_status_to_string(rc)); } core_teardown_after_shutdown(spl); + // Free the sealed live log's extents now that the cache is flushed, before + // the map is persisted (Part B) so the persisted map reflects the free. + log_dec_ref(spl->cc, &live_log); /* * Release the context's live root reference before persisting the map, so * the persisted refcounts reflect exactly the durable record's single @@ -2385,7 +2519,11 @@ core_destroy(core_handle *spl) } } + // Discard the live log too: seal (frees the handle), then free its extents + // after the cache flush. + log_segment_info live_log = core_seal_live_log(spl); core_teardown_after_shutdown(spl); + log_dec_ref(spl->cc, &live_log); trunk_context_deinit(&spl->trunk_context); /* diff --git a/src/core.h b/src/core.h index 440a2aef..21c10727 100644 --- a/src/core.h +++ b/src/core.h @@ -252,6 +252,13 @@ core_mount(core_handle *spl, allocator_root_id id, platform_heap_id hid); +/* + * Take a checkpoint: advance the durable root and rotate the log (two-log + * protocol). Must be called at a quiescent point (no concurrent inserts). + */ +platform_status +core_checkpoint(core_handle *spl); + platform_status core_unmount(core_handle *spl); diff --git a/src/log.h b/src/log.h index 5cba7244..aca1756f 100644 --- a/src/log.h +++ b/src/log.h @@ -96,3 +96,12 @@ log_get_segment_info(log_handle *log) log_handle * log_create(cache *cc, log_config *cfg, platform_heap_id hid); + +/* + * Release a sealed log segment identified by its log_segment_info: drop the + * reference its metadata extent holds, freeing the segment's on-disk extents. + * Takes no handle -- the handle was freed by log_seal(); the caller retained + * only the identity (log_get_segment_info(), captured at creation). + */ +void +log_dec_ref(cache *cc, const log_segment_info *segment); diff --git a/src/shard_log.c b/src/shard_log.c index 862a4465..40878537 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -356,7 +356,7 @@ shard_log_write(log_handle *logh, * the mini-allocator's unused reserve and frees the handle. After seal the * handle is invalid; the caller retains the identity it captured earlier * (shard_log_get_segment_info(), fixed at creation) to reopen the stream - * for replay and, eventually, to free its extents via shard_log_dec_ref(). + * for replay and, eventually, to free its extents via log_dec_ref(). * * The caller must prevent concurrent shard_log_write() and seal calls. * seal itself issues no writeback or durable barrier: to make the sealed @@ -412,7 +412,7 @@ shard_log_seal(log_handle *logh) * The stream is now immutable. Release the mini-allocator's unused * per-batch reserve so no future allocation touches this stream, and free * the handle. The caller already holds the stream's identity (captured at - * creation) and later frees the on-disk extents via shard_log_dec_ref(). + * creation) and later frees the on-disk extents via log_dec_ref(). */ mini_release(&log->mini); platform_free(log->heap_id, log); @@ -420,7 +420,7 @@ shard_log_seal(log_handle *logh) } void -shard_log_dec_ref(cache *cc, const log_segment_info *segment) +log_dec_ref(cache *cc, const log_segment_info *segment) { if (segment->meta_addr == 0) { return; diff --git a/src/shard_log.h b/src/shard_log.h index 32c1d171..b3c599df 100644 --- a/src/shard_log.h +++ b/src/shard_log.h @@ -76,14 +76,6 @@ typedef struct ONDISK shard_log_hdr { uint16 num_entries; } shard_log_hdr; -/* - * Release a sealed log segment: drop the mini-allocator reference its metadata - * extent holds, freeing the segment's on-disk extents. Identified by - * log_segment_info (no handle needed -- the handle was freed by log_seal()). - */ -void -shard_log_dec_ref(cache *cc, const log_segment_info *segment); - platform_status shard_log_iterator_init(cache *cc, shard_log_config *cfg, diff --git a/tests/functional/log_test.c b/tests/functional/log_test.c index f1c4ec6c..bb033400 100644 --- a/tests/functional/log_test.c +++ b/tests/functional/log_test.c @@ -143,7 +143,7 @@ test_log_crash(clockcache *cc, merge_accumulator_deinit(&msg); shard_log_iterator_deinit(hid, &itor); - shard_log_dec_ref((cache *)cc, &segment); + log_dec_ref((cache *)cc, &segment); return 0; } @@ -301,8 +301,8 @@ test_log_two_segments(clockcache *cc, test_log_verify_segment( (cache *)cc, cfg, &fresh, gen, hid, key_size, new_first, new_count); - shard_log_dec_ref((cache *)cc, &sealed); - shard_log_dec_ref((cache *)cc, &fresh); + log_dec_ref((cache *)cc, &sealed); + log_dec_ref((cache *)cc, &fresh); return 0; } @@ -370,7 +370,7 @@ test_log_large_message(cache *cc, shard_log_iterator_deinit(hid, &itor); merge_accumulator_deinit(&msg); - shard_log_dec_ref(cc, &sealed); + log_dec_ref(cc, &sealed); return 0; } @@ -462,7 +462,7 @@ test_log_perf(cache *cc, cleanup: // Seal (frees the handle) and release the segment's extents. log_seal(logh); - shard_log_dec_ref((cache *)cc, &sealed); + log_dec_ref((cache *)cc, &sealed); platform_free(hid, params); return ret; diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index e7f9e2ee..3196c861 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -251,6 +251,66 @@ CTEST2(splinter, test_inserts) core_destroy(&spl); } +/* + * With logging enabled, core_checkpoint() rotates the log via the two-log + * protocol and advances the durable root. Data inserted before the checkpoint + * must survive it, a second (empty) checkpoint must be a clean rotate, and + * teardown's allocator_assert_noleaks() must pass (the sealed log's extents are + * freed, the root is not leaked or double-freed). + */ +CTEST2(splinter, test_two_log_checkpoint) +{ + allocator *alp = (allocator *)&data->al; + data->system_cfg->splinter_cfg.use_log = TRUE; // exercise the two-log lifecycle + + core_handle spl; + platform_status rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + test_generate_allocator_root_id(), + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + uint64 num_inserts = splinter_do_inserts(data, &spl, FALSE, NULL); + ASSERT_NOT_EQUAL(0, num_inserts); + + // Checkpoint: seal the live log into the sealed slot, start a fresh live + // log, incorporate, advance the durable root, then clear + free the sealed + // log. + rc = core_checkpoint(&spl); + ASSERT_TRUE(SUCCESS(rc)); + + // A sample of keys must still be found after the checkpoint. + lookup_result qdata; + lookup_result_init( + &qdata, spl.cfg.data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + const size_t key_size = data->workload_cfg->key_size; + uint64 verify_count = (num_inserts < 1000) ? num_inserts : 1000; + for (uint64 i = 0; i < verify_count; i++) { + test_key(&keybuf, TEST_RANDOM, i, 0, 0, key_size, 0); + rc = core_lookup(&spl, key_buffer_key(&keybuf), &qdata); + ASSERT_TRUE(SUCCESS(rc)); + verify_tuple( + &spl, + &data->gen, + i, + key_buffer_key(&keybuf), + merge_accumulator_to_message(lookup_result_accumulator(&qdata)), + TRUE); + } + lookup_result_deinit(&qdata); + + // Second checkpoint with no new inserts is a clean rotate. + rc = core_checkpoint(&spl); + ASSERT_TRUE(SUCCESS(rc)); + + core_destroy(&spl); +} + /* * The second checkpoint slot is a torn-write fallback, not permission for a * normal mount to silently roll back past a newer, valid active record. A diff --git a/tests/unit/splinterdb_quick_test.c b/tests/unit/splinterdb_quick_test.c index 5dee4428..59536c1e 100644 --- a/tests/unit/splinterdb_quick_test.c +++ b/tests/unit/splinterdb_quick_test.c @@ -1251,6 +1251,48 @@ CTEST2(splinterdb_quick, test_close_and_reopen) splinterdb_lookup_result_deinit(&result); } +/* + * With logging enabled, the two-log lifecycle records the live log in the + * superblock at create/mount and discards it (seal + dec_ref) at a clean close. + * A reopen must still find data inserted before the close. + */ +CTEST2(splinterdb_quick, test_logged_close_and_reopen) +{ + // Re-create the instance with logging enabled (SETUP created it without). + splinterdb_close(&data->kvsb); + data->cfg.use_log = TRUE; + int rc = splinterdb_create(&data->cfg, &data->kvsb); + ASSERT_EQUAL(0, rc); + + slice user_key = slice_create(strlen("logged-key"), "logged-key"); + const char *val = "logged-value"; + const size_t val_len = strlen(val); + rc = splinterdb_insert(data->kvsb, user_key, slice_create(val_len, val), NULL); + ASSERT_EQUAL(0, rc); + + splinterdb_close(&data->kvsb); + rc = splinterdb_open(&data->cfg, &data->kvsb); + ASSERT_EQUAL(0, rc); + + splinterdb_lookup_result result; + splinterdb_lookup_result_init( + data->kvsb, &result, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + rc = splinterdb_lookup(data->kvsb, user_key, &result); + ASSERT_EQUAL(0, rc); + ASSERT_TRUE(splinterdb_lookup_found(&result)); + + slice value; + rc = splinterdb_lookup_result_value(&result, &value); + ASSERT_EQUAL(0, rc); + ASSERT_EQUAL(val_len, slice_length(value)); + ASSERT_STREQN(val, + slice_data(value), + slice_length(value), + "logged reopen value mismatch up to %d bytes\n", + val_len); + splinterdb_lookup_result_deinit(&result); +} + /* * Regression test for bug where repeating a cycle of insert-close-reopen * causes a space leak and eventually hits an assertion From 902cf19b3b82de00de341b127c63973cd1aba79b Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sun, 26 Jul 2026 09:59:15 -0700 Subject: [PATCH 24/64] comment cleanup Signed-off-by: Rob Johnson --- src/core.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/core.c b/src/core.c index 12e9b3be..fd3cfd54 100644 --- a/src/core.c +++ b/src/core.c @@ -767,11 +767,6 @@ core_memtable_flush_internal_virtual(task *arg) /* * Function to trigger a memtable incorporation. Called in the context of * the foreground doing insertions. - * If background threads are not enabled, this function does the entire memtable - * incorporation inline. - * If background threads are enabled, this function just queues up the task to - * carry out the incorporation, swaps the curr_memtable pointer, claims the - * root and returns. */ static void core_memtable_flush(core_handle *spl, uint64 generation) @@ -2303,7 +2298,8 @@ core_teardown_after_shutdown(core_handle *spl) memtable_context_deinit(&spl->mt_ctxt); // flush all dirty pages in the cache. The live log has already been sealed - // by the caller (core_seal_live_log); its extents are freed after this flush. + // by the caller (core_seal_live_log); its extents are freed after this + // flush. cache_flush(spl->cc); } From 5d0346b6df8255c8a95e0d54f18333fea8b399ef Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sun, 26 Jul 2026 18:05:09 -0700 Subject: [PATCH 25/64] incorporation-driven checkpointing Signed-off-by: Rob Johnson --- src/core.c | 258 +++++++++++++++++++++++++++++++++++-- src/core.h | 54 +++++++- src/memtable.c | 8 ++ src/memtable.h | 9 ++ tests/unit/splinter_test.c | 86 +++++++++++++ 5 files changed, 406 insertions(+), 9 deletions(-) diff --git a/src/core.c b/src/core.c index fd3cfd54..ea94bdde 100644 --- a/src/core.c +++ b/src/core.c @@ -51,21 +51,36 @@ core_checkpoint_lock_init(core_handle *spl) { platform_status rc = platform_mutex_init( &spl->checkpoint_lock, platform_get_module_id(), spl->heap_id); - if (SUCCESS(rc)) { - spl->checkpoint_lock_initialized = TRUE; + if (!SUCCESS(rc)) { + return rc; } - return rc; + spl->checkpoint_lock_initialized = TRUE; + + rc = platform_mutex_init( + &spl->checkpoint_state_lock, platform_get_module_id(), spl->heap_id); + if (!SUCCESS(rc)) { + return rc; + } + spl->checkpoint_state_lock_initialized = TRUE; + + ZERO_CONTENTS(&spl->checkpoint); // phase == CORE_CHECKPOINT_IDLE + spl->last_checkpoint_generation = 0; + return STATUS_OK; } static void core_checkpoint_lock_deinit(core_handle *spl) { - if (!spl->checkpoint_lock_initialized) { - return; + if (spl->checkpoint_state_lock_initialized) { + platform_status rc = platform_mutex_destroy(&spl->checkpoint_state_lock); + platform_assert_status_ok(rc); + spl->checkpoint_state_lock_initialized = FALSE; + } + if (spl->checkpoint_lock_initialized) { + platform_status rc = platform_mutex_destroy(&spl->checkpoint_lock); + platform_assert_status_ok(rc); + spl->checkpoint_lock_initialized = FALSE; } - platform_status rc = platform_mutex_destroy(&spl->checkpoint_lock); - platform_assert_status_ok(rc); - spl->checkpoint_lock_initialized = FALSE; } /* @@ -307,6 +322,211 @@ core_publish_root_record(core_handle *spl, return rc; } +/* + *----------------------------------------------------------------------------- + * Incorporation-driven checkpoint (two-log protocol) + * + * A checkpoint rotates the log and advances the durable root without stopping + * the world. It is driven off memtable rotation and incorporation: + * + * begin (rotation): pre-create the next live log outside the critical + * section, swap it in under the insert lock (so no writer can be + * mid-write to the old log), then seal the old log just after. + * complete (incorporation): once the sealed log's generations are folded + * into the trunk root, publish the advanced root with the sealed + * slot cleared and free the sealed log's extents. + * + * See core_checkpoint_state in core.h for the phase machine and its locking. + *----------------------------------------------------------------------------- + */ + +/* Policy: should the next memtable rotation start a checkpoint? */ +static bool32 +core_should_take_checkpoint(core_handle *spl) +{ + if (!spl->cfg.use_log || spl->cfg.checkpoint_generation_interval == 0) { + return FALSE; + } + uint64 current = spl->mt_ctxt.generation; + return current - spl->last_checkpoint_generation + >= spl->cfg.checkpoint_generation_interval; +} + +/* + * Begin, step 1 (outside the rotation critical section): if no checkpoint is in + * progress and policy says so, pre-create the next live log and arm the swap. + * log_create does no disk I/O, but is kept off the insert-blocking path. + */ +static void +core_checkpoint_maybe_begin(core_handle *spl) +{ + platform_mutex_lock(&spl->checkpoint_state_lock); + bool32 begin = spl->checkpoint.phase == CORE_CHECKPOINT_IDLE + && core_should_take_checkpoint(spl); + platform_mutex_unlock(&spl->checkpoint_state_lock); + if (!begin) { + return; + } + + log_handle *next = log_create(spl->cc, spl->cfg.log_cfg, spl->heap_id); + if (next == NULL) { + platform_error_log( + "core_checkpoint_maybe_begin: log_create failed; skipping\n"); + return; + } + log_segment_info next_info = log_get_segment_info(next); + + platform_mutex_lock(&spl->checkpoint_state_lock); + if (spl->checkpoint.phase == CORE_CHECKPOINT_IDLE) { + spl->checkpoint.pending_log = next; + spl->checkpoint.live_info = next_info; + spl->checkpoint.phase = CORE_CHECKPOINT_PENDING; + spl->last_checkpoint_generation = spl->mt_ctxt.generation; + next = NULL; // handed off to the checkpoint + } + platform_mutex_unlock(&spl->checkpoint_state_lock); + + if (next != NULL) { + // Lost a race with a concurrent rotation; discard the speculative log. + log_seal(next); + log_dec_ref(spl->cc, &next_info); + } +} + +/* + * Begin, step 2 (inside the rotation critical section, insert lock held + * exclusively): swap the pre-created live log in. Registered as + * mt_ctxt.rotate. Every log writer holds the insert lock shared across its + * log_write, so once this store retires no writer is mid-write to, or will + * newly enter, the old log -- making the subsequent seal safe. + */ +static void +core_rotate_log_virtual(void *arg, uint64 finalized_generation) +{ + core_handle *spl = arg; + + platform_mutex_lock(&spl->checkpoint_state_lock); + if (spl->checkpoint.phase == CORE_CHECKPOINT_PENDING) { + spl->checkpoint.log_to_seal = spl->log; + spl->checkpoint.sealed_info = log_get_segment_info(spl->log); + spl->log = spl->checkpoint.pending_log; + spl->checkpoint.pending_log = NULL; + spl->checkpoint.cut_generation = finalized_generation; + spl->checkpoint.phase = CORE_CHECKPOINT_SEALING; + } + platform_mutex_unlock(&spl->checkpoint_state_lock); +} + +/* + * Begin, step 3 (just after the rotation critical section): seal the swapped-out + * log. The swap drained and excluded all writers, so sealing is safe. Seal + * finalizes the pages (dirty) and frees the handle; durability is deferred to + * completion (or a future sync), so no barrier here. + */ +static void +core_checkpoint_seal_cut(core_handle *spl) +{ + platform_mutex_lock(&spl->checkpoint_state_lock); + log_handle *to_seal = NULL; + if (spl->checkpoint.phase == CORE_CHECKPOINT_SEALING) { + to_seal = spl->checkpoint.log_to_seal; + spl->checkpoint.log_to_seal = NULL; + spl->checkpoint.phase = CORE_CHECKPOINT_INCORPORATING; + } + platform_mutex_unlock(&spl->checkpoint_state_lock); + + if (to_seal != NULL) { + log_seal(to_seal); + } +} + +/* + * Complete (after an incorporation): if the sealed log's generations are all + * folded into the trunk root, advance the durable root with the sealed slot + * cleared (which makes the root and superblock durable) and free the sealed + * log's extents. Called from the single-threaded incorporation path. + */ +static platform_status +core_maybe_complete_checkpoint(core_handle *spl) +{ + platform_mutex_lock(&spl->checkpoint_state_lock); + uint64 retired = memtable_generation_retired(&spl->mt_ctxt); + bool32 complete = + spl->checkpoint.phase == CORE_CHECKPOINT_INCORPORATING + && retired != SUPERBLOCK_NO_INCORPORATED_GENERATION + && retired >= spl->checkpoint.cut_generation; + log_segment_info sealed = {0}; + log_segment_info live = {0}; + if (complete) { + sealed = spl->checkpoint.sealed_info; + live = spl->checkpoint.live_info; + spl->checkpoint.phase = CORE_CHECKPOINT_COMPLETING; + } + platform_mutex_unlock(&spl->checkpoint_state_lock); + + if (!complete) { + return STATUS_OK; + } + + platform_status rc = core_publish_root_record( + spl, core_log_to_superblock(live), (superblock_log_info){0}); + if (SUCCESS(rc)) { + // The sealed log's entries are now durably in the root; free its extents. + log_dec_ref(spl->cc, &sealed); + } else { + platform_error_log( + "core_maybe_complete_checkpoint: publish failed: %s\n", + platform_status_to_string(rc)); + } + + platform_mutex_lock(&spl->checkpoint_state_lock); + if (SUCCESS(rc)) { + ZERO_CONTENTS(&spl->checkpoint.sealed_info); + ZERO_CONTENTS(&spl->checkpoint.live_info); + spl->checkpoint.cut_generation = 0; + spl->checkpoint.phase = CORE_CHECKPOINT_IDLE; + } else { + // Leave the sealed slot intact and retry on a later incorporation. + spl->checkpoint.phase = CORE_CHECKPOINT_INCORPORATING; + } + platform_mutex_unlock(&spl->checkpoint_state_lock); + return rc; +} + +/* + * Resolve any in-flight checkpoint during a quiesced shutdown, before the + * unmount/destroy publish. No locking: the caller has quiesced all inserts and + * incorporations. A completed checkpoint (INCORPORATING/COMPLETING) is normally + * already reaped by the quiesce drain; the residual cases below are defensive. + */ +static void +core_finish_checkpoint_for_shutdown(core_handle *spl) +{ + core_checkpoint_state *cp = &spl->checkpoint; + switch (cp->phase) { + case CORE_CHECKPOINT_IDLE: + break; + case CORE_CHECKPOINT_PENDING: + // The next live log was pre-created but never installed; discard it. + log_seal(cp->pending_log); + log_dec_ref(spl->cc, &cp->live_info); + break; + case CORE_CHECKPOINT_INCORPORATING: + case CORE_CHECKPOINT_COMPLETING: + // The sealed log is fully incorporated after quiesce. The shutdown + // publish records sealed=none, so just reclaim its extents here + // (before the map is persisted, so the map reflects the free). + log_dec_ref(spl->cc, &cp->sealed_info); + break; + case CORE_CHECKPOINT_SEALING: + default: + platform_assert(FALSE, + "unexpected checkpoint phase %d at shutdown", + cp->phase); + } + ZERO_CONTENTS(cp); // phase == CORE_CHECKPOINT_IDLE +} + /* *----------------------------------------------------------------------------- * Memtable Functions @@ -748,6 +968,9 @@ core_memtable_flush_internal(core_handle *spl, uint64 generation) } generation++; } while (core_try_continue_incorporate(spl, generation)); + + // A checkpoint's sealed log may now be fully incorporated; complete it. + core_maybe_complete_checkpoint(spl); out: return STATUS_OK; } @@ -785,7 +1008,16 @@ static void core_memtable_flush_virtual(void *arg, uint64 generation) { core_handle *spl = arg; + + // Begin, step 3: if this rotation's in-CS hook swapped the live log, seal + // the old one now that the critical section has been released. + core_checkpoint_seal_cut(spl); + core_memtable_flush(spl, generation); + + // Begin, step 1: decide whether the next rotation should start a checkpoint, + // pre-creating its live log outside any critical section. + core_checkpoint_maybe_begin(spl); } static inline uint64 @@ -1977,6 +2209,8 @@ core_mkfs(core_handle *spl, platform_status_to_string(rc)); goto deinit_superblock; } + // Swap the checkpoint's live log in from inside the rotation critical section. + spl->mt_ctxt.rotate = core_rotate_log_virtual; // set up the log if (spl->cfg.use_log) { @@ -2137,6 +2371,8 @@ core_mount(core_handle *spl, platform_status_to_string(rc)); goto deinit_superblock; } + // Swap the checkpoint's live log in from inside the rotation critical section. + spl->mt_ctxt.rotate = core_rotate_log_virtual; if (spl->cfg.use_log) { spl->log = log_create(cc, spl->cfg.log_cfg, spl->heap_id); @@ -2429,6 +2665,9 @@ core_unmount(core_handle *spl) */ core_quiesce_for_shutdown(spl); + // Reclaim any in-flight checkpoint's logs before the unmount publish. + core_finish_checkpoint_for_shutdown(spl); + /* * The clean-unmount root incorporates everything, so the live log is fully * folded and discardable. Seal it (frees the handle) now; free its extents @@ -2495,6 +2734,9 @@ core_destroy(core_handle *spl) { core_quiesce_for_shutdown(spl); + // Reclaim any in-flight checkpoint's logs before teardown. + core_finish_checkpoint_for_shutdown(spl); + /* * Release the reference the published tree record holds on its root before * tearing down the context (which releases the context's own live root diff --git a/src/core.h b/src/core.h index 21c10727..a1fadfd3 100644 --- a/src/core.h +++ b/src/core.h @@ -51,7 +51,11 @@ typedef struct core_config { data_config *data_cfg; bool32 use_log; log_config *log_cfg; - trunk_config *trunk_node_cfg; + // Automatic-checkpoint policy: take a checkpoint (rotate the log and advance + // the durable root) once this many memtable generations have been finalized + // since the last one. 0 disables automatic checkpoints. + uint64 checkpoint_generation_interval; + trunk_config *trunk_node_cfg; // verbose logging bool32 verbose_logging_enabled; @@ -94,6 +98,43 @@ typedef struct core_branch { typedef struct core_handle core_handle; +/* + * Incorporation-driven checkpoint (two-log protocol) state machine. + * + * IDLE no checkpoint in progress. + * PENDING the next live log is pre-created; the next memtable rotation + * will swap it in under the insert lock. + * SEALING the rotation swapped the new live log in; the old log still + * needs sealing (done just after the rotation critical section). + * INCORPORATING the old log is sealed; waiting for its generations to be + * incorporated into the trunk root. + * COMPLETING the completion publish (advance root, clear sealed slot) is + * in flight. + * + * The only transition that touches the shared spl->log pointer (PENDING -> + * SEALING) runs inside the memtable rotation critical section, where the insert + * lock is held exclusively; every log writer holds that lock shared across its + * log_write, so no writer can be mid-write to, or newly enter, the old log once + * it is swapped out. All other fields are guarded by checkpoint_state_lock, + * which is only ever held for brief, I/O-free updates. + */ +typedef enum core_checkpoint_phase { + CORE_CHECKPOINT_IDLE = 0, + CORE_CHECKPOINT_PENDING, + CORE_CHECKPOINT_SEALING, + CORE_CHECKPOINT_INCORPORATING, + CORE_CHECKPOINT_COMPLETING, +} core_checkpoint_phase; + +typedef struct core_checkpoint_state { + core_checkpoint_phase phase; + log_handle *pending_log; // next live log, pre-created (PENDING) + log_handle *log_to_seal; // old live log awaiting seal (SEALING) + log_segment_info sealed_info; // identity of the sealed log (reclaim) + log_segment_info live_info; // identity of the new live log + uint64 cut_generation; // complete once retired >= this +} core_checkpoint_state; + typedef struct core_memtable_args { core_handle *spl; uint64 generation; @@ -133,6 +174,17 @@ struct core_handle { platform_mutex checkpoint_lock; bool32 checkpoint_lock_initialized; + /* + * Incorporation-driven checkpoint state. checkpoint_state_lock guards the + * fields of `checkpoint` and last_checkpoint_generation; it is only ever + * held for brief, I/O-free updates (never across a barrier), so taking it + * inside the memtable rotation critical section cannot stall inserts on I/O. + */ + platform_mutex checkpoint_state_lock; + bool32 checkpoint_state_lock_initialized; + core_checkpoint_state checkpoint; + uint64 last_checkpoint_generation; + core_stats *stats; core_compacted_memtable compacted_memtable[MAX_MEMTABLES]; diff --git a/src/memtable.c b/src/memtable.c index e8b8969b..9b537b20 100644 --- a/src/memtable.c +++ b/src/memtable.c @@ -193,6 +193,14 @@ memtable_maybe_rotate_and_begin_insert(memtable_context *ctxt, current_generation); memtable_mark_empty(ctxt); + /* + * Still holding the insert lock exclusively: all in-flight inserts + * (and their log writes) have drained and none can start. This is + * the one safe point to swap the checkpoint's live log. + */ + if (ctxt->rotate != NULL) { + ctxt->rotate(ctxt->process_ctxt, current_generation); + } memtable_end_insert_rotation(ctxt); memtable_end_insert(ctxt); memtable_process(ctxt, current_generation); diff --git a/src/memtable.h b/src/memtable.h index bfea7002..f0351115 100644 --- a/src/memtable.h +++ b/src/memtable.h @@ -127,6 +127,15 @@ typedef struct memtable_context { process_fn process; void *process_ctxt; + /* + * Optional callback invoked inside the rotation critical section (insert + * lock held exclusively), after a memtable is finalized and the generation + * is advanced, before the lock is released. A checkpoint uses this to swap + * in a new live log with the guarantee that no insert can be mid-log_write. + * Receives process_ctxt and the just-finalized generation. NULL disables. + */ + process_fn rotate; + // batch distributed read/write locks protect the generation and // generation_retired counters batch_rwlock rwlock; diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index 3196c861..956744f7 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -311,6 +311,92 @@ CTEST2(splinter, test_two_log_checkpoint) core_destroy(&spl); } +/* + * Shared workload for the automatic-checkpoint tests: create with auto + * checkpoints enabled, insert enough to drive several rotations, verify a + * sample of keys survives the mid-run log rotations, then destroy. The + * teardown's allocator_assert_noleaks() must pass -- each sealed log's extents + * are freed as its checkpoint completes, and the root is neither leaked nor + * double-freed. The caller sets up the task system (foreground or background). + */ +static void +run_auto_checkpoint_workload(void *datap, uint64 interval) +{ + struct CTEST_IMPL_DATA_SNAME(splinter) *data = + (struct CTEST_IMPL_DATA_SNAME(splinter) *)datap; + + allocator *alp = (allocator *)&data->al; + data->system_cfg->splinter_cfg.use_log = TRUE; + // Rotate the log / advance the durable root every `interval` generations. + data->system_cfg->splinter_cfg.checkpoint_generation_interval = interval; + + core_handle spl; + platform_status rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + test_generate_allocator_root_id(), + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + uint64 num_inserts = splinter_do_inserts(data, &spl, FALSE, NULL); + ASSERT_NOT_EQUAL(0, num_inserts); + + // Drain so any in-flight checkpoint completes and the state settles. + rc = task_perform_until_quiescent(spl.ts); + ASSERT_TRUE(SUCCESS(rc)); + + if (interval != 0) { + // The inserts must have driven at least one automatic checkpoint. + ASSERT_NOT_EQUAL(0, spl.last_checkpoint_generation); + + // At rest a checkpoint has either completed (IDLE) or been armed for a + // rotation that idle never triggered (PENDING); both leave no sealed log. + ASSERT_TRUE(spl.checkpoint.phase == CORE_CHECKPOINT_IDLE + || spl.checkpoint.phase == CORE_CHECKPOINT_PENDING); + superblock_tree_record rec; + superblock_get_tree_record(&spl.superblock, &rec); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(rec.sealed_log)); + // A checkpoint published an advanced, incorporated durable root mid-run. + ASSERT_NOT_EQUAL(SUPERBLOCK_NO_INCORPORATED_GENERATION, + rec.incorporated_generation); + } + + // A sample of keys must still be found after the rotations. + lookup_result qdata; + lookup_result_init( + &qdata, spl.cfg.data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + const size_t key_size = data->workload_cfg->key_size; + uint64 verify_count = (num_inserts < 1000) ? num_inserts : 1000; + for (uint64 i = 0; i < verify_count; i++) { + test_key(&keybuf, TEST_RANDOM, i, 0, 0, key_size, 0); + rc = core_lookup(&spl, key_buffer_key(&keybuf), &qdata); + ASSERT_TRUE(SUCCESS(rc)); + verify_tuple( + &spl, + &data->gen, + i, + key_buffer_key(&keybuf), + merge_accumulator_to_message(lookup_result_accumulator(&qdata)), + TRUE); + } + lookup_result_deinit(&qdata); + + core_destroy(&spl); +} + +/* + * Automatic, incorporation-driven checkpoints (foreground): the log is rotated + * and the durable root advanced during normal inserts, with no stop-the-world. + */ +CTEST2(splinter, test_auto_checkpoint) +{ + run_auto_checkpoint_workload(data, 2); +} + /* * The second checkpoint slot is a torn-write fallback, not permission for a * normal mount to silently roll back past a newer, valid active record. A From 4361da60ecfad4239138f74c0714dd0b108cabe9 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Sun, 26 Jul 2026 22:56:57 -0700 Subject: [PATCH 26/64] formatting Signed-off-by: Rob Johnson --- src/core.c | 38 +++++++++++++------------- src/core.h | 15 +++++----- src/log.h | 3 +- src/mini_allocator.c | 22 +++++++-------- src/shard_log.h | 9 +++--- src/superblock.c | 35 ++++++++++++------------ src/trunk.c | 10 +++++-- tests/functional/log_test.c | 10 ++----- tests/unit/splinter_test.c | 5 ++-- tests/unit/splinterdb_quick_test.c | 3 +- tests/unit/superblock_test.c | 44 ++++++++++++++++++------------ 11 files changed, 103 insertions(+), 91 deletions(-) diff --git a/src/core.c b/src/core.c index ea94bdde..a47bc3e1 100644 --- a/src/core.c +++ b/src/core.c @@ -418,10 +418,10 @@ core_rotate_log_virtual(void *arg, uint64 finalized_generation) } /* - * Begin, step 3 (just after the rotation critical section): seal the swapped-out - * log. The swap drained and excluded all writers, so sealing is safe. Seal - * finalizes the pages (dirty) and frees the handle; durability is deferred to - * completion (or a future sync), so no barrier here. + * Begin, step 3 (just after the rotation critical section): seal the + * swapped-out log. The swap drained and excluded all writers, so sealing is + * safe. Seal finalizes the pages (dirty) and frees the handle; durability is + * deferred to completion (or a future sync), so no barrier here. */ static void core_checkpoint_seal_cut(core_handle *spl) @@ -450,11 +450,10 @@ static platform_status core_maybe_complete_checkpoint(core_handle *spl) { platform_mutex_lock(&spl->checkpoint_state_lock); - uint64 retired = memtable_generation_retired(&spl->mt_ctxt); - bool32 complete = - spl->checkpoint.phase == CORE_CHECKPOINT_INCORPORATING - && retired != SUPERBLOCK_NO_INCORPORATED_GENERATION - && retired >= spl->checkpoint.cut_generation; + uint64 retired = memtable_generation_retired(&spl->mt_ctxt); + bool32 complete = spl->checkpoint.phase == CORE_CHECKPOINT_INCORPORATING + && retired != SUPERBLOCK_NO_INCORPORATED_GENERATION + && retired >= spl->checkpoint.cut_generation; log_segment_info sealed = {0}; log_segment_info live = {0}; if (complete) { @@ -474,9 +473,8 @@ core_maybe_complete_checkpoint(core_handle *spl) // The sealed log's entries are now durably in the root; free its extents. log_dec_ref(spl->cc, &sealed); } else { - platform_error_log( - "core_maybe_complete_checkpoint: publish failed: %s\n", - platform_status_to_string(rc)); + platform_error_log("core_maybe_complete_checkpoint: publish failed: %s\n", + platform_status_to_string(rc)); } platform_mutex_lock(&spl->checkpoint_state_lock); @@ -496,8 +494,9 @@ core_maybe_complete_checkpoint(core_handle *spl) /* * Resolve any in-flight checkpoint during a quiesced shutdown, before the * unmount/destroy publish. No locking: the caller has quiesced all inserts and - * incorporations. A completed checkpoint (INCORPORATING/COMPLETING) is normally - * already reaped by the quiesce drain; the residual cases below are defensive. + * incorporations. A completed checkpoint (INCORPORATING/COMPLETING) is + * normally already reaped by the quiesce drain; the residual cases below are + * defensive. */ static void core_finish_checkpoint_for_shutdown(core_handle *spl) @@ -520,9 +519,8 @@ core_finish_checkpoint_for_shutdown(core_handle *spl) break; case CORE_CHECKPOINT_SEALING: default: - platform_assert(FALSE, - "unexpected checkpoint phase %d at shutdown", - cp->phase); + platform_assert( + FALSE, "unexpected checkpoint phase %d at shutdown", cp->phase); } ZERO_CONTENTS(cp); // phase == CORE_CHECKPOINT_IDLE } @@ -2209,7 +2207,8 @@ core_mkfs(core_handle *spl, platform_status_to_string(rc)); goto deinit_superblock; } - // Swap the checkpoint's live log in from inside the rotation critical section. + // Swap the checkpoint's live log in from inside the rotation critical + // section. spl->mt_ctxt.rotate = core_rotate_log_virtual; // set up the log @@ -2371,7 +2370,8 @@ core_mount(core_handle *spl, platform_status_to_string(rc)); goto deinit_superblock; } - // Swap the checkpoint's live log in from inside the rotation critical section. + // Swap the checkpoint's live log in from inside the rotation critical + // section. spl->mt_ctxt.rotate = core_rotate_log_virtual; if (spl->cfg.use_log) { diff --git a/src/core.h b/src/core.h index a1fadfd3..7dc76394 100644 --- a/src/core.h +++ b/src/core.h @@ -105,11 +105,10 @@ typedef struct core_handle core_handle; * PENDING the next live log is pre-created; the next memtable rotation * will swap it in under the insert lock. * SEALING the rotation swapped the new live log in; the old log still - * needs sealing (done just after the rotation critical section). - * INCORPORATING the old log is sealed; waiting for its generations to be - * incorporated into the trunk root. - * COMPLETING the completion publish (advance root, clear sealed slot) is - * in flight. + * needs sealing (done just after the rotation critical + * section). INCORPORATING the old log is sealed; waiting for its generations to + * be incorporated into the trunk root. COMPLETING the completion publish + * (advance root, clear sealed slot) is in flight. * * The only transition that touches the shared spl->log pointer (PENDING -> * SEALING) runs inside the memtable rotation critical section, where the insert @@ -130,9 +129,9 @@ typedef struct core_checkpoint_state { core_checkpoint_phase phase; log_handle *pending_log; // next live log, pre-created (PENDING) log_handle *log_to_seal; // old live log awaiting seal (SEALING) - log_segment_info sealed_info; // identity of the sealed log (reclaim) - log_segment_info live_info; // identity of the new live log - uint64 cut_generation; // complete once retired >= this + log_segment_info sealed_info; // identity of the sealed log (reclaim) + log_segment_info live_info; // identity of the new live log + uint64 cut_generation; // complete once retired >= this } core_checkpoint_state; typedef struct core_memtable_args { diff --git a/src/log.h b/src/log.h index aca1756f..37fda13f 100644 --- a/src/log.h +++ b/src/log.h @@ -87,7 +87,8 @@ log_seal(log_handle *log) return log->ops->seal(log); } -/* The stream's durable identity (fixed at creation). See log_segment_info_fn. */ +/* The stream's durable identity (fixed at creation). See log_segment_info_fn. + */ static inline log_segment_info log_get_segment_info(log_handle *log) { diff --git a/src/mini_allocator.c b/src/mini_allocator.c index 6832604e..a4f328d6 100644 --- a/src/mini_allocator.c +++ b/src/mini_allocator.c @@ -788,9 +788,9 @@ typedef platform_status (*mini_meta_extent_fn)(cache *cc, void *arg); typedef platform_status (*mini_for_each_meta_page_fn)(cache *cc, - page_type type, - page_handle *meta_page, - void *arg); + page_type type, + page_handle *meta_page, + void *arg); static platform_status mini_for_each_meta_page(cache *cc, @@ -1110,8 +1110,8 @@ mini_recover_visit_meta_page(cache *cc, page_handle *meta_page, void *arg) { - mini_recover_state *state = (mini_recover_state *)arg; - uint64 meta_addr = meta_page->disk_addr; + mini_recover_state *state = (mini_recover_state *)arg; + uint64 meta_addr = meta_page->disk_addr; if (state->page_count == state->max_meta_pages) { return mini_recovery_corruption("metadata chain exceeds disk pages", @@ -1209,12 +1209,12 @@ mini_recover_allocations(cache *cc, uint64 meta_head, page_type meta_type) return mini_recovery_corruption("metadata head is not a page", meta_head); } - mini_recover_state state = {.al = al, - .cfg = cfg, - .expected_prev = 0, - .page_count = 0, - .max_meta_pages = cfg->capacity - / cfg->io_cfg->page_size}; + mini_recover_state state = {.al = al, + .cfg = cfg, + .expected_prev = 0, + .page_count = 0, + .max_meta_pages = + cfg->capacity / cfg->io_cfg->page_size}; return mini_for_each_meta_page(cc, meta_head, diff --git a/src/shard_log.h b/src/shard_log.h index b3c599df..d53e4f0f 100644 --- a/src/shard_log.h +++ b/src/shard_log.h @@ -38,10 +38,11 @@ typedef struct shard_log_thread_data { * Sharded log context structure. */ typedef struct shard_log { - log_handle super; // handle to log I/O ops abstraction. - cache *cc; - shard_log_config *cfg; - platform_heap_id heap_id; // heap the handle was allocated from; freed by seal + log_handle super; // handle to log I/O ops abstraction. + cache *cc; + shard_log_config *cfg; + platform_heap_id + heap_id; // heap the handle was allocated from; freed by seal shard_log_thread_data thread_data[MAX_THREADS]; mini_allocator mini; uint64 addr; diff --git a/src/superblock.c b/src/superblock.c index d7d85168..283c72c5 100644 --- a/src/superblock.c +++ b/src/superblock.c @@ -42,8 +42,7 @@ superblock_is_valid(const superblock *sb, const allocator_config *cfg) return sb->format_magic == SUPERBLOCK_FORMAT_MAGIC && sb->format_version == SUPERBLOCK_FORMAT_VERSION && superblock_geometry_matches(geometry, cfg) - && platform_checksum_is_equal(sb->checksum, - superblock_checksum(sb)); + && platform_checksum_is_equal(sb->checksum, superblock_checksum(sb)); } platform_status @@ -70,7 +69,8 @@ superblock_context_init(superblock_context *ctx, platform_assert(sizeof(superblock) <= ctx->page_size); - platform_status rc = platform_buffer_init(&ctx->image_buffer, ctx->page_size); + platform_status rc = + platform_buffer_init(&ctx->image_buffer, ctx->page_size); if (!SUCCESS(rc)) { return rc; } @@ -103,8 +103,7 @@ superblock_mount(superblock_context *ctx, const allocator_config *cfg) bool32 have_valid = FALSE; for (uint64 s = 0; s < SUPERBLOCK_NUM_SLOTS; s++) { - rc = io_read( - ctx->io, slot, ctx->page_size, superblock_slot_addr(ctx, s)); + rc = io_read(ctx->io, slot, ctx->page_size, superblock_slot_addr(ctx, s)); if (!SUCCESS(rc)) { goto out; } @@ -126,12 +125,12 @@ superblock_mount(superblock_context *ctx, const allocator_config *cfg) rc = STATUS_OK; out: - { - platform_status deinit_rc = platform_buffer_deinit(&slot_buffer); - if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { - rc = deinit_rc; - } +{ + platform_status deinit_rc = platform_buffer_deinit(&slot_buffer); + if (SUCCESS(rc) && !SUCCESS(deinit_rc)) { + rc = deinit_rc; } +} return rc; } @@ -152,7 +151,7 @@ superblock_publish(superblock_context *ctx) { platform_assert(ctx->image != NULL); - uint64 target = ctx->current_slot ^ 1; + uint64 target = ctx->current_slot ^ 1; ctx->image->generation += 1; platform_assert(ctx->image->generation != 0); // generation wraparound @@ -174,12 +173,12 @@ superblock_format(superblock_context *ctx, const allocator_config *cfg) platform_assert(ctx->image != NULL); memset(ctx->image, 0, ctx->page_size); - ctx->image->geometry.disk_size = cfg->capacity; - ctx->image->geometry.page_size = cfg->io_cfg->page_size; - ctx->image->geometry.extent_size = cfg->io_cfg->extent_size; - ctx->image->format_magic = SUPERBLOCK_FORMAT_MAGIC; - ctx->image->format_version = SUPERBLOCK_FORMAT_VERSION; - ctx->image->generation = 0; + ctx->image->geometry.disk_size = cfg->capacity; + ctx->image->geometry.page_size = cfg->io_cfg->page_size; + ctx->image->geometry.extent_size = cfg->io_cfg->extent_size; + ctx->image->format_magic = SUPERBLOCK_FORMAT_MAGIC; + ctx->image->format_version = SUPERBLOCK_FORMAT_VERSION; + ctx->image->generation = 0; ctx->image->allocation_state_addr = 0; // fresh DB: rebuild on crash // A fresh, empty tree: no root and nothing incorporated yet. ctx->image->tree.incorporated_generation = @@ -191,7 +190,7 @@ superblock_format(superblock_context *ctx, const allocator_config *cfg) * initial current_slot is 1; the second targets slot 1 (generation 2), * leaving slot 1 newest. */ - ctx->current_slot = 1; + ctx->current_slot = 1; platform_status rc = superblock_publish(ctx); if (!SUCCESS(rc)) { return rc; diff --git a/src/trunk.c b/src/trunk.c index 4ea8bdb9..e5b7c9ab 100644 --- a/src/trunk.c +++ b/src/trunk.c @@ -2587,9 +2587,13 @@ trunk_snapshot_release(trunk_context *context, trunk_snapshot *snapshot) * share context's root. */ trunk_context scratch; - platform_status rc = trunk_context_init( - &scratch, context->cfg, context->hid, context->cc, context->al, - context->ts, *snapshot); + platform_status rc = trunk_context_init(&scratch, + context->cfg, + context->hid, + context->cc, + context->al, + context->ts, + *snapshot); snapshot->root_addr = 0; // trunk_context_init consumes it regardless of rc if (!SUCCESS(rc)) { return rc; diff --git a/tests/functional/log_test.c b/tests/functional/log_test.c index bb033400..034d67e8 100644 --- a/tests/functional/log_test.c +++ b/tests/functional/log_test.c @@ -307,9 +307,7 @@ test_log_two_segments(clockcache *cc, } static int -test_log_large_message(cache *cc, - shard_log_config *cfg, - platform_heap_id hid) +test_log_large_message(cache *cc, shard_log_config *cfg, platform_heap_id hid) { platform_status rc; log_segment_info sealed; @@ -332,8 +330,7 @@ test_log_large_message(cache *cc, merge_accumulator_set_class(&msg, MESSAGE_TYPE_INSERT); memset(merge_accumulator_data(&msg), 'L', value_len); - int log_rc = - log_write(logh, skey, merge_accumulator_to_message(&msg), 0, 0); + int log_rc = log_write(logh, skey, merge_accumulator_to_message(&msg), 0, 0); platform_assert(log_rc == 0); merge_accumulator filler; @@ -357,8 +354,7 @@ test_log_large_message(cache *cc, rc = cache_durable_barrier(cc); platform_assert_status_ok(rc); - rc = shard_log_iterator_init( - cc, cfg, hid, sealed.addr, sealed.magic, &itor); + rc = shard_log_iterator_init(cc, cfg, hid, sealed.addr, sealed.magic, &itor); platform_assert_status_ok(rc); platform_assert(iterator_can_curr(itorh)); diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index 956744f7..3b481cc1 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -261,7 +261,8 @@ CTEST2(splinter, test_inserts) CTEST2(splinter, test_two_log_checkpoint) { allocator *alp = (allocator *)&data->al; - data->system_cfg->splinter_cfg.use_log = TRUE; // exercise the two-log lifecycle + data->system_cfg->splinter_cfg.use_log = + TRUE; // exercise the two-log lifecycle core_handle spl; platform_status rc = core_mkfs(&spl, @@ -325,7 +326,7 @@ run_auto_checkpoint_workload(void *datap, uint64 interval) struct CTEST_IMPL_DATA_SNAME(splinter) *data = (struct CTEST_IMPL_DATA_SNAME(splinter) *)datap; - allocator *alp = (allocator *)&data->al; + allocator *alp = (allocator *)&data->al; data->system_cfg->splinter_cfg.use_log = TRUE; // Rotate the log / advance the durable root every `interval` generations. data->system_cfg->splinter_cfg.checkpoint_generation_interval = interval; diff --git a/tests/unit/splinterdb_quick_test.c b/tests/unit/splinterdb_quick_test.c index 59536c1e..512b902b 100644 --- a/tests/unit/splinterdb_quick_test.c +++ b/tests/unit/splinterdb_quick_test.c @@ -1267,7 +1267,8 @@ CTEST2(splinterdb_quick, test_logged_close_and_reopen) slice user_key = slice_create(strlen("logged-key"), "logged-key"); const char *val = "logged-value"; const size_t val_len = strlen(val); - rc = splinterdb_insert(data->kvsb, user_key, slice_create(val_len, val), NULL); + rc = + splinterdb_insert(data->kvsb, user_key, slice_create(val_len, val), NULL); ASSERT_EQUAL(0, rc); splinterdb_close(&data->kvsb); diff --git a/tests/unit/superblock_test.c b/tests/unit/superblock_test.c index a07008b3..2eb9fed6 100644 --- a/tests/unit/superblock_test.c +++ b/tests/unit/superblock_test.c @@ -100,7 +100,8 @@ CTEST2(superblock, test_format_sets_fresh_state) rec.incorporated_generation); superblock_context_deinit(&ctx); - rc = superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); ASSERT_TRUE(SUCCESS(rc)); rc = superblock_mount(&ctx, &data->allocator_cfg); ASSERT_TRUE(SUCCESS(rc)); @@ -126,7 +127,7 @@ CTEST2(superblock, test_publish_persists_tree_record) superblock_tree_record rec = { .root_addr = 0x4000, .incorporated_generation = SUPERBLOCK_NO_INCORPORATED_GENERATION, - .live_log = {.addr = 0x6000, .meta_addr = 0x8000, .magic = 0x11}, + .live_log = {.addr = 0x6000, .meta_addr = 0x8000, .magic = 0x11}, }; superblock_set_tree_record(&ctx, &rec); superblock_set_allocation_state_addr(&ctx, 0x8000); @@ -134,7 +135,8 @@ CTEST2(superblock, test_publish_persists_tree_record) ASSERT_TRUE(SUCCESS(rc)); superblock_context_deinit(&ctx); - rc = superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); ASSERT_TRUE(SUCCESS(rc)); rc = superblock_mount(&ctx, &data->allocator_cfg); ASSERT_TRUE(SUCCESS(rc)); @@ -152,15 +154,18 @@ CTEST2(superblock, test_publish_persists_tree_record) } /* Steady, begin-checkpoint, and complete-checkpoint tree-record states. */ -static const superblock_log_info TEST_LOG_L1 = { - .addr = 0x6000, .meta_addr = 0x8000, .magic = 0x11}; -static const superblock_log_info TEST_LOG_L2 = { - .addr = 0x10000, .meta_addr = 0x12000, .magic = 0x22}; +static const superblock_log_info TEST_LOG_L1 = {.addr = 0x6000, + .meta_addr = 0x8000, + .magic = 0x11}; +static const superblock_log_info TEST_LOG_L2 = {.addr = 0x10000, + .meta_addr = 0x12000, + .magic = 0x22}; /* * Walk the two-log checkpoint state machine through the superblock and confirm * each published state reads back: steady {root R0, live L1, no sealed} -> - * begin {root R0, sealed L1, live L2} -> complete {root R1, live L2, no sealed}. + * begin {root R0, sealed L1, live L2} -> complete {root R1, live L2, no + * sealed}. */ CTEST2(superblock, test_two_log_checkpoint_transitions) { @@ -202,7 +207,8 @@ CTEST2(superblock, test_two_log_checkpoint_transitions) superblock_context_deinit(&ctx); // A fresh mount reads the completed state. - rc = superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); ASSERT_TRUE(SUCCESS(rc)); rc = superblock_mount(&ctx, &data->allocator_cfg); ASSERT_TRUE(SUCCESS(rc)); @@ -228,8 +234,9 @@ CTEST2(superblock, test_two_log_checkpoint_torn_begin) rc = superblock_format(&ctx, &data->allocator_cfg); // slot0=gen1, slot1=gen2 ASSERT_TRUE(SUCCESS(rc)); - // Publish the steady state into BOTH slots (gen3->slot0, gen4->slot1), so the - // fallback below is unambiguously the steady state, not the empty format one. + // Publish the steady state into BOTH slots (gen3->slot0, gen4->slot1), so + // the fallback below is unambiguously the steady state, not the empty format + // one. superblock_tree_record rec = { .root_addr = 0x4000, .incorporated_generation = 5, @@ -253,7 +260,8 @@ CTEST2(superblock, test_two_log_checkpoint_torn_begin) superblock_test_corrupt_slot(data->ioh, data->io_cfg.page_size, 0); // Mount falls back to slot1 (gen4) = steady: live L1, no sealed. - rc = superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); ASSERT_TRUE(SUCCESS(rc)); rc = superblock_mount(&ctx, &data->allocator_cfg); ASSERT_TRUE(SUCCESS(rc)); @@ -293,10 +301,11 @@ CTEST2(superblock, test_torn_write_falls_back_to_older_generation) // Simulate a torn write of the newest slot (slot 0, gen 3). superblock_test_corrupt_slot(data->ioh, data->io_cfg.page_size, 0); - // Mount must still succeed, falling back to slot 1 (gen 2), which carries the - // format's empty tree (root_addr 0), not the published root -- proving the - // older generation was left intact by the torn write. - rc = superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + // Mount must still succeed, falling back to slot 1 (gen 2), which carries + // the format's empty tree (root_addr 0), not the published root -- proving + // the older generation was left intact by the torn write. + rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); ASSERT_TRUE(SUCCESS(rc)); rc = superblock_mount(&ctx, &data->allocator_cfg); ASSERT_TRUE(SUCCESS(rc)); @@ -321,7 +330,8 @@ CTEST2(superblock, test_both_slots_corrupt_is_not_found) superblock_test_corrupt_slot(data->ioh, data->io_cfg.page_size, 0); superblock_test_corrupt_slot(data->ioh, data->io_cfg.page_size, 1); - rc = superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); ASSERT_TRUE(SUCCESS(rc)); rc = superblock_mount(&ctx, &data->allocator_cfg); ASSERT_FALSE(SUCCESS(rc)); From 8957f665f7f8f0248de85e856e61fc66baa9dcb3 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Mon, 27 Jul 2026 21:46:48 -0700 Subject: [PATCH 27/64] convert superblock to semantic transitions Signed-off-by: Rob Johnson --- src/core.c | 104 ++++++++++++++++------------------- src/superblock.c | 49 +++++++++++++---- src/superblock.h | 71 ++++++++++++++++++------ tests/unit/superblock_test.c | 91 +++++++++++++----------------- 4 files changed, 175 insertions(+), 140 deletions(-) diff --git a/src/core.c b/src/core.c index a47bc3e1..d4a8f300 100644 --- a/src/core.c +++ b/src/core.c @@ -182,19 +182,18 @@ core_log_to_superblock(log_segment_info info) } /* - * Publish a tree-record root advance to the superblock: capture the current - * COW root, make it durable, record it (with the incorporated-generation cut - * and the given live/sealed log pointers), invalidate the persisted allocation - * state in the same atomic update, and release the previously published root. - * Used by mkfs (empty root, records the new live log) and by unmount (Part A; - * unmount then persists the map and republishes a valid allocation state). The - * snapshot's owned reference is transferred to the durable record on a - * successful publish. + * Advance the durable tree to the current COW root and make it durable: capture + * the root, make its pages durable, snapshot it into the superblock (with the + * incorporated-generation cut, the given live log carried forward, and the + * sealed slot cleared -- a snapshot happens only after the sealed log is folded + * in), then release the previously published root. snapshot_tree invalidates + * the persisted allocation state; a clean unmount revalidates it in a later + * step. Used by mkfs (empty root, records the new live log), checkpoint + * completion, and unmount (Part A). The snapshot's owned reference is + * transferred to the durable record on success. */ static platform_status -core_publish_root_record(core_handle *spl, - superblock_log_info live_log, - superblock_log_info sealed_log) +core_publish_root_record(core_handle *spl, superblock_log_info live_log) { platform_status rc; trunk_snapshot snapshot; @@ -202,7 +201,6 @@ core_publish_root_record(core_handle *spl, uint64 incorporated_generation; uint64 old_root_addr = 0; superblock_tree_record old_rec; - superblock_tree_record rec; /* * The snapshot cut, durable record write, and old-root release are one @@ -239,29 +237,26 @@ core_publish_root_record(core_handle *spl, superblock_get_tree_record(&spl->superblock, &old_rec); old_root_addr = old_rec.root_addr; - ZERO_CONTENTS(&rec); - rec.root_addr = snapshot.root_addr; - rec.incorporated_generation = has_incorporated_generation - ? incorporated_generation - : SUPERBLOCK_NO_INCORPORATED_GENERATION; - rec.live_log = live_log; - rec.sealed_log = sealed_log; - - superblock_set_tree_record(&spl->superblock, &rec); /* - * Advancing a root invalidates the persisted allocation map: the in-memory - * map now diverges from disk. A clean unmount republishes a valid - * allocation state only after persisting the map (Part B). + * Snapshot the new root (clearing the sealed slot and carrying live_log + * forward) and make it durable. snapshot_tree invalidates the persisted + * allocation map -- the in-memory map now diverges from disk; a clean unmount + * revalidates it only after persisting the map (Part B). */ - superblock_set_allocation_state_addr(&spl->superblock, 0); - - rc = superblock_publish(&spl->superblock); + superblock_snapshot_tree(&spl->superblock, + snapshot.root_addr, + has_incorporated_generation + ? incorporated_generation + : SUPERBLOCK_NO_INCORPORATED_GENERATION, + live_log); + + rc = superblock_make_durable(&spl->superblock); if (!SUCCESS(rc)) { /* The old root is still the newest durable one; keep its reference. */ goto release_snapshot; } - if (old_root_addr == rec.root_addr) { + if (old_root_addr == snapshot.root_addr) { /* * Republishing the same root (e.g. a clean unmount with no advance since * the last publish): the previously published record already accounts @@ -467,8 +462,8 @@ core_maybe_complete_checkpoint(core_handle *spl) return STATUS_OK; } - platform_status rc = core_publish_root_record( - spl, core_log_to_superblock(live), (superblock_log_info){0}); + platform_status rc = + core_publish_root_record(spl, core_log_to_superblock(live)); if (SUCCESS(rc)) { // The sealed log's entries are now durably in the root; free its extents. log_dec_ref(spl->cc, &sealed); @@ -2247,7 +2242,7 @@ core_mkfs(core_handle *spl, if (spl->cfg.use_log) { live_log = core_log_to_superblock(log_get_segment_info(spl->log)); } - rc = core_publish_root_record(spl, live_log, (superblock_log_info){0}); + rc = core_publish_root_record(spl, live_log); if (!SUCCESS(rc)) { platform_error_log("core_mkfs: core_publish_root_record failed: %s\n", platform_status_to_string(rc)); @@ -2413,23 +2408,20 @@ core_mount(core_handle *spl, } /* - * Mark dirty: record this session's fresh live log and invalidate the - * persisted allocation state, before any allocation diverges the in-memory - * map from disk. A crash after this forces the next mount into recovery - * instead of silently reverting to this now-stale root. The root is - * unchanged; a clean mount has no sealed log. + * Mark dirty: cut this session's fresh live log and invalidate the persisted + * allocation state, before any allocation diverges the in-memory map from + * disk. A crash after this forces the next mount into recovery instead of + * silently reverting to this now-stale root. The root is unchanged; a clean + * mount has no prior live log, so the sealed slot stays empty. */ - superblock_get_tree_record(&spl->superblock, &rec); - rec.sealed_log = (superblock_log_info){0}; - rec.live_log = spl->cfg.use_log - ? core_log_to_superblock(log_get_segment_info(spl->log)) - : (superblock_log_info){0}; - superblock_set_tree_record(&spl->superblock, &rec); - superblock_set_allocation_state_addr(&spl->superblock, 0); - rc = superblock_publish(&spl->superblock); + superblock_log_cut( + &spl->superblock, + spl->cfg.use_log ? core_log_to_superblock(log_get_segment_info(spl->log)) + : (superblock_log_info){0}); + rc = superblock_make_durable(&spl->superblock); if (!SUCCESS(rc)) { - platform_error_log("core_mount: mark-dirty superblock_publish failed: " - "%s\n", + platform_error_log("core_mount: mark-dirty superblock_make_durable " + "failed: %s\n", platform_status_to_string(rc)); goto deinit_stats; } @@ -2605,14 +2597,11 @@ core_checkpoint(core_handle *spl) } } if (SUCCESS(rc)) { + // Cut the log: the just-sealed live log becomes the sealed slot and + // the fresh log becomes live. Root unchanged until completion. new_live = core_log_to_superblock(log_get_segment_info(spl->log)); - superblock_tree_record rec; - superblock_get_tree_record(&spl->superblock, &rec); - rec.live_log = new_live; - rec.sealed_log = core_log_to_superblock(sealed); - superblock_set_tree_record(&spl->superblock, &rec); - superblock_set_allocation_state_addr(&spl->superblock, 0); - rc = superblock_publish(&spl->superblock); + superblock_log_cut(&spl->superblock, new_live); + rc = superblock_make_durable(&spl->superblock); } platform_status unlock_rc = platform_mutex_unlock(&spl->checkpoint_lock); @@ -2637,7 +2626,7 @@ core_checkpoint(core_handle *spl) // --- Complete: advance the durable root to the incorporated state, keeping // the new live log and clearing the sealed slot. --- - rc = core_publish_root_record(spl, new_live, (superblock_log_info){0}); + rc = core_publish_root_record(spl, new_live); if (!SUCCESS(rc)) { return rc; } @@ -2680,8 +2669,7 @@ core_unmount(core_handle *spl) * live or sealed log at rest). Allocation state stays invalid here; it * becomes valid only in Part B, after the map is persisted. */ - rc = core_publish_root_record( - spl, (superblock_log_info){0}, (superblock_log_info){0}); + rc = core_publish_root_record(spl, (superblock_log_info){0}); if (!SUCCESS(rc)) { platform_error_log("core_unmount: failed to publish unmount root: %s\n", platform_status_to_string(rc)); @@ -2709,8 +2697,8 @@ core_unmount(core_handle *spl) uint64 map_addr; platform_status prc = allocator_persist(spl->al, &map_addr); if (SUCCESS(prc)) { - superblock_set_allocation_state_addr(&spl->superblock, map_addr); - prc = superblock_publish(&spl->superblock); + superblock_snapshot_allocator(&spl->superblock, map_addr); + prc = superblock_make_durable(&spl->superblock); } if (!SUCCESS(prc)) { platform_error_log("core_unmount: failed to publish clean allocation " diff --git a/src/superblock.c b/src/superblock.c index 283c72c5..b7a925b8 100644 --- a/src/superblock.c +++ b/src/superblock.c @@ -136,7 +136,7 @@ superblock_mount(superblock_context *ctx, const allocator_config *cfg) /* * Checksum the in-memory image and write it to a physical slot. Does not make - * it durable; superblock_publish() issues the barrier. + * it durable; superblock_make_durable() issues the barrier. */ static platform_status superblock_write_slot(superblock_context *ctx, uint64 slot) @@ -147,7 +147,7 @@ superblock_write_slot(superblock_context *ctx, uint64 slot) } platform_status -superblock_publish(superblock_context *ctx) +superblock_make_durable(superblock_context *ctx) { platform_assert(ctx->image != NULL); @@ -191,11 +191,11 @@ superblock_format(superblock_context *ctx, const allocator_config *cfg) * leaving slot 1 newest. */ ctx->current_slot = 1; - platform_status rc = superblock_publish(ctx); + platform_status rc = superblock_make_durable(ctx); if (!SUCCESS(rc)) { return rc; } - return superblock_publish(ctx); + return superblock_make_durable(ctx); } bool32 @@ -211,21 +211,46 @@ superblock_allocation_state_addr(const superblock_context *ctx) } void -superblock_set_allocation_state_addr(superblock_context *ctx, uint64 addr) +superblock_get_tree_record(const superblock_context *ctx, + superblock_tree_record *out) { - ctx->image->allocation_state_addr = addr; + *out = ctx->image->tree; } void -superblock_get_tree_record(const superblock_context *ctx, - superblock_tree_record *out) +superblock_log_cut(superblock_context *ctx, superblock_log_info new_live) { - *out = ctx->image->tree; + // The current live log becomes the sealed log (its entries are being folded + // into the next root); new_live receives subsequent inserts. The persisted + // allocation map no longer matches the (log) state, so invalidate it. + ctx->image->tree.sealed_log = ctx->image->tree.live_log; + ctx->image->tree.live_log = new_live; + ctx->image->allocation_state_addr = 0; +} + +void +superblock_snapshot_tree(superblock_context *ctx, + uint64 root_addr, + uint64 incorporated_generation, + superblock_log_info new_live) +{ + // The durable tree now includes everything folded into root_addr, so the + // sealed log (if any) is done with; new_live is the log carried forward + // (empty at a clean shutdown). Advancing the root diverges the persisted + // allocation map, so invalidate it. + ctx->image->tree.root_addr = root_addr; + ctx->image->tree.incorporated_generation = incorporated_generation; + ctx->image->tree.sealed_log = (superblock_log_info){0}; + ctx->image->tree.live_log = new_live; + ctx->image->allocation_state_addr = 0; } void -superblock_set_tree_record(superblock_context *ctx, - const superblock_tree_record *rec) +superblock_snapshot_allocator(superblock_context *ctx, uint64 map_addr) { - ctx->image->tree = *rec; + // The only operation that validates the allocation state; every tree/log + // transition invalidates it. The caller must have made the map itself + // durable first, then make the superblock durable after. + platform_assert(map_addr != 0); + ctx->image->allocation_state_addr = map_addr; } diff --git a/src/superblock.h b/src/superblock.h index c10cfffc..35d8aa78 100644 --- a/src/superblock.h +++ b/src/superblock.h @@ -161,16 +161,62 @@ platform_status superblock_format(superblock_context *ctx, const allocator_config *cfg); /* - * Publish the current in-memory image: bump the generation, write it to the - * physical slot not currently newest, and make it durable. This is the single - * atomic commit for a checkpoint or a clean-unmount transition; mutate the - * image first via the setters below (tree records, allocation state), then - * publish. + * ---- Durable-state transitions ---- + * + * Each transition mutates the in-memory image only; nothing reaches disk until + * superblock_make_durable(). A checkpoint or clean unmount is a sequence of + * these transitions followed by one superblock_make_durable(), so the + * durability boundary is always explicit at the call site. + * + * The tree/log transitions encode the crash-safety invariant that the persisted + * allocation map is trustworthy only while it matches the durable tree: + * superblock_log_cut() and superblock_snapshot_tree() both invalidate it, and + * superblock_snapshot_allocator() is the only operation that re-validates it. + */ + +/* + * Rotate the log: the current live log becomes the sealed log (its entries are + * being folded into the next root) and new_live receives subsequent inserts. + * Invalidates the allocation state. Used at a checkpoint's begin, and to + * install a fresh live log at mkfs/mount (where there is no prior live log, so + * the sealed slot stays empty). + */ +void +superblock_log_cut(superblock_context *ctx, superblock_log_info new_live); + +/* + * Advance the durable tree to root_addr (having incorporated up to + * incorporated_generation) and clear the sealed log -- a snapshot is taken only + * after the sealed log has been folded into the root. new_live is the log + * carried forward (empty at a clean shutdown). Invalidates the allocation + * state. Used at a checkpoint's completion and at a clean unmount. + */ +void +superblock_snapshot_tree(superblock_context *ctx, + uint64 root_addr, + uint64 incorporated_generation, + superblock_log_info new_live); + +/* + * Record the persisted allocator refcount map at map_addr as trustworthy. This + * is the only operation that validates the allocation state; the caller must + * have made the map itself durable first. Used as the final step of a clean + * unmount. + */ +void +superblock_snapshot_allocator(superblock_context *ctx, uint64 map_addr); + +/* + * Make the current in-memory image durable: bump the generation, write it to + * the physical slot not currently newest, and issue a durable barrier. On + * success that slot becomes newest; a torn write leaves the previous generation + * intact in the other slot. This is the single durability boundary for the + * transitions above. */ platform_status -superblock_publish(superblock_context *ctx); +superblock_make_durable(superblock_context *ctx); -/* ---- Accessors on the in-memory image ---- */ +/* ---- Read-only accessors on the in-memory image ---- */ bool32 superblock_allocation_state_valid(const superblock_context *ctx); @@ -178,18 +224,7 @@ superblock_allocation_state_valid(const superblock_context *ctx); uint64 superblock_allocation_state_addr(const superblock_context *ctx); -void -superblock_set_allocation_state_addr(superblock_context *ctx, uint64 addr); - /* Copy the (always-present) tree record into *out. */ void superblock_get_tree_record(const superblock_context *ctx, superblock_tree_record *out); - -/* - * Store rec as the tree record. In-memory only; not durable until - * superblock_publish(). - */ -void -superblock_set_tree_record(superblock_context *ctx, - const superblock_tree_record *rec); diff --git a/tests/unit/superblock_test.c b/tests/unit/superblock_test.c index 2eb9fed6..b71747d4 100644 --- a/tests/unit/superblock_test.c +++ b/tests/unit/superblock_test.c @@ -112,10 +112,11 @@ CTEST2(superblock, test_format_sets_fresh_state) } /* - * A published tree record and allocation state survive a deinit + re-mount: - * the newest generation wins. + * A durable tree snapshot and a subsequent durable allocator snapshot (the + * clean-unmount Part A / Part B sequence) survive a deinit + re-mount: the + * newest generation wins. */ -CTEST2(superblock, test_publish_persists_tree_record) +CTEST2(superblock, test_snapshot_persists_state) { superblock_context ctx; platform_status rc = @@ -124,14 +125,16 @@ CTEST2(superblock, test_publish_persists_tree_record) rc = superblock_format(&ctx, &data->allocator_cfg); ASSERT_TRUE(SUCCESS(rc)); - superblock_tree_record rec = { - .root_addr = 0x4000, - .incorporated_generation = SUPERBLOCK_NO_INCORPORATED_GENERATION, - .live_log = {.addr = 0x6000, .meta_addr = 0x8000, .magic = 0x11}, - }; - superblock_set_tree_record(&ctx, &rec); - superblock_set_allocation_state_addr(&ctx, 0x8000); - rc = superblock_publish(&ctx); + superblock_log_info live = { + .addr = 0x6000, .meta_addr = 0x8000, .magic = 0x11}; + superblock_snapshot_tree( + &ctx, 0x4000, SUPERBLOCK_NO_INCORPORATED_GENERATION, live); + rc = superblock_make_durable(&ctx); + ASSERT_TRUE(SUCCESS(rc)); + ASSERT_FALSE(superblock_allocation_state_valid(&ctx)); // snapshot invalidated + + superblock_snapshot_allocator(&ctx, 0x8000); + rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); superblock_context_deinit(&ctx); @@ -176,20 +179,14 @@ CTEST2(superblock, test_two_log_checkpoint_transitions) rc = superblock_format(&ctx, &data->allocator_cfg); ASSERT_TRUE(SUCCESS(rc)); - superblock_tree_record rec = { - .root_addr = 0x4000, - .incorporated_generation = 5, - .live_log = TEST_LOG_L1, - }; - superblock_set_tree_record(&ctx, &rec); - rc = superblock_publish(&ctx); + // Steady: root R0, live L1, no sealed log. + superblock_snapshot_tree(&ctx, 0x4000, 5, TEST_LOG_L1); + rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); - // Begin: seal L1 -> sealed_log, new live L2. - rec.sealed_log = rec.live_log; - rec.live_log = TEST_LOG_L2; - superblock_set_tree_record(&ctx, &rec); - rc = superblock_publish(&ctx); + // Begin: cut the log -- L1 becomes sealed, L2 becomes live. + superblock_log_cut(&ctx, TEST_LOG_L2); + rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); superblock_tree_record got; @@ -197,12 +194,9 @@ CTEST2(superblock, test_two_log_checkpoint_transitions) ASSERT_EQUAL(TEST_LOG_L1.meta_addr, got.sealed_log.meta_addr); ASSERT_EQUAL(TEST_LOG_L2.meta_addr, got.live_log.meta_addr); - // Complete: advance root, clear sealed_log. - rec.root_addr = 0x4400; - rec.incorporated_generation = 9; - rec.sealed_log = (superblock_log_info){0}; - superblock_set_tree_record(&ctx, &rec); - rc = superblock_publish(&ctx); + // Complete: advance root, carry L2 forward, clear sealed_log. + superblock_snapshot_tree(&ctx, 0x4400, 9, TEST_LOG_L2); + rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); superblock_context_deinit(&ctx); @@ -234,25 +228,18 @@ CTEST2(superblock, test_two_log_checkpoint_torn_begin) rc = superblock_format(&ctx, &data->allocator_cfg); // slot0=gen1, slot1=gen2 ASSERT_TRUE(SUCCESS(rc)); - // Publish the steady state into BOTH slots (gen3->slot0, gen4->slot1), so - // the fallback below is unambiguously the steady state, not the empty format - // one. - superblock_tree_record rec = { - .root_addr = 0x4000, - .incorporated_generation = 5, - .live_log = TEST_LOG_L1, - }; - superblock_set_tree_record(&ctx, &rec); - rc = superblock_publish(&ctx); // gen3 -> slot0 + // Make the steady state durable into BOTH slots (gen3->slot0, gen4->slot1), + // so the fallback below is unambiguously the steady state, not the empty + // format one. + superblock_snapshot_tree(&ctx, 0x4000, 5, TEST_LOG_L1); + rc = superblock_make_durable(&ctx); // gen3 -> slot0 ASSERT_TRUE(SUCCESS(rc)); - rc = superblock_publish(&ctx); // gen4 -> slot1 + rc = superblock_make_durable(&ctx); // gen4 -> slot1 ASSERT_TRUE(SUCCESS(rc)); - // Begin checkpoint: this publish (gen5) targets slot0. - rec.sealed_log = rec.live_log; - rec.live_log = TEST_LOG_L2; - superblock_set_tree_record(&ctx, &rec); - rc = superblock_publish(&ctx); // gen5 -> slot0 + // Begin checkpoint: cut the log; this make_durable (gen5) targets slot0. + superblock_log_cut(&ctx, TEST_LOG_L2); + rc = superblock_make_durable(&ctx); // gen5 -> slot0 ASSERT_TRUE(SUCCESS(rc)); superblock_context_deinit(&ctx); @@ -288,13 +275,13 @@ CTEST2(superblock, test_torn_write_falls_back_to_older_generation) rc = superblock_format(&ctx, &data->allocator_cfg); ASSERT_TRUE(SUCCESS(rc)); - // After format the image is gen 2 in slot 1, so this publish targets slot 0. - superblock_tree_record rec = { - .root_addr = 0x4000, - .incorporated_generation = SUPERBLOCK_NO_INCORPORATED_GENERATION, - }; - superblock_set_tree_record(&ctx, &rec); - rc = superblock_publish(&ctx); + // After format the image is gen 2 in slot 1, so this make_durable targets + // slot 0. Snapshot a nonempty root with no live log. + superblock_snapshot_tree(&ctx, + 0x4000, + SUPERBLOCK_NO_INCORPORATED_GENERATION, + (superblock_log_info){0}); + rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); superblock_context_deinit(&ctx); From e84b01471bc6383e22b10099b8cde6cfbb9e1970 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Mon, 27 Jul 2026 22:00:43 -0700 Subject: [PATCH 28/64] comment Signed-off-by: Rob Johnson --- src/superblock.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/superblock.h b/src/superblock.h index 35d8aa78..ea62391c 100644 --- a/src/superblock.h +++ b/src/superblock.h @@ -141,6 +141,8 @@ superblock_context_init(superblock_context *ctx, const allocator_config *cfg, platform_heap_id hid); +/* Release in-memory resources. Does no I/O. Specifically, does _not_ make the + * current in-memory superblock durable. */ void superblock_context_deinit(superblock_context *ctx); From 69acd491f6c94ae233877620e9b7ed00e7d481da Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Mon, 27 Jul 2026 22:00:49 -0700 Subject: [PATCH 29/64] formatting Signed-off-by: Rob Johnson --- src/core.c | 4 ++-- tests/unit/superblock_test.c | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core.c b/src/core.c index d4a8f300..45bb74aa 100644 --- a/src/core.c +++ b/src/core.c @@ -240,8 +240,8 @@ core_publish_root_record(core_handle *spl, superblock_log_info live_log) /* * Snapshot the new root (clearing the sealed slot and carrying live_log * forward) and make it durable. snapshot_tree invalidates the persisted - * allocation map -- the in-memory map now diverges from disk; a clean unmount - * revalidates it only after persisting the map (Part B). + * allocation map -- the in-memory map now diverges from disk; a clean + * unmount revalidates it only after persisting the map (Part B). */ superblock_snapshot_tree(&spl->superblock, snapshot.root_addr, diff --git a/tests/unit/superblock_test.c b/tests/unit/superblock_test.c index b71747d4..bb53d7da 100644 --- a/tests/unit/superblock_test.c +++ b/tests/unit/superblock_test.c @@ -131,7 +131,8 @@ CTEST2(superblock, test_snapshot_persists_state) &ctx, 0x4000, SUPERBLOCK_NO_INCORPORATED_GENERATION, live); rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); - ASSERT_FALSE(superblock_allocation_state_valid(&ctx)); // snapshot invalidated + ASSERT_FALSE( + superblock_allocation_state_valid(&ctx)); // snapshot invalidated superblock_snapshot_allocator(&ctx, 0x8000); rc = superblock_make_durable(&ctx); From f1821e58ad97860cda47b7fa8e1141178d04699e Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Mon, 27 Jul 2026 22:02:54 -0700 Subject: [PATCH 30/64] formatting Signed-off-by: Rob Johnson --- src/superblock.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/superblock.h b/src/superblock.h index ea62391c..2474bd35 100644 --- a/src/superblock.h +++ b/src/superblock.h @@ -150,6 +150,8 @@ superblock_context_deinit(superblock_context *ctx); * Read both physical copies, validate each (magic, version, checksum, and * geometry against cfg), and load the newest valid one into the in-memory * image. Returns STATUS_NOT_FOUND if neither copy is a valid superblock. + * + * Must call superblock_init() first. */ platform_status superblock_mount(superblock_context *ctx, const allocator_config *cfg); @@ -158,6 +160,8 @@ superblock_mount(superblock_context *ctx, const allocator_config *cfg); * Initialize a fresh superblock in the in-memory image (empty tree table, * allocation state invalid) and write both physical copies durably. Used by * mkfs. + * + * Must call superblock_init() first. */ platform_status superblock_format(superblock_context *ctx, const allocator_config *cfg); From 5b846a6308702822fe0dbe2fff2c4717eaaafc31 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Tue, 28 Jul 2026 02:14:27 -0700 Subject: [PATCH 31/64] Cleanup log Signed-off-by: Rob Johnson --- src/core.c | 23 ++++++----- src/log.h | 79 +++++++++++++++++++++++++++++++++++-- src/shard_log.c | 74 ++++++++++++++++++++++++++-------- src/shard_log.h | 46 +++++++++++---------- tests/functional/log_test.c | 70 +++++++++++++++----------------- 5 files changed, 200 insertions(+), 92 deletions(-) diff --git a/src/core.c b/src/core.c index 45bb74aa..b99c71c7 100644 --- a/src/core.c +++ b/src/core.c @@ -8,6 +8,7 @@ */ #include "core.h" +#include "shard_log.h" // core constructs the concrete log via shard_log_create() #include "data_internal.h" #include "notification.h" #include "platform_sleep.h" @@ -350,7 +351,7 @@ core_should_take_checkpoint(core_handle *spl) /* * Begin, step 1 (outside the rotation critical section): if no checkpoint is in * progress and policy says so, pre-create the next live log and arm the swap. - * log_create does no disk I/O, but is kept off the insert-blocking path. + * Log creation does no disk I/O, but is kept off the insert-blocking path. */ static void core_checkpoint_maybe_begin(core_handle *spl) @@ -363,10 +364,11 @@ core_checkpoint_maybe_begin(core_handle *spl) return; } - log_handle *next = log_create(spl->cc, spl->cfg.log_cfg, spl->heap_id); + log_handle *next = shard_log_create( + spl->cc, (shard_log_config *)spl->cfg.log_cfg, spl->heap_id); if (next == NULL) { platform_error_log( - "core_checkpoint_maybe_begin: log_create failed; skipping\n"); + "core_checkpoint_maybe_begin: shard_log_create failed; skipping\n"); return; } log_segment_info next_info = log_get_segment_info(next); @@ -2208,9 +2210,10 @@ core_mkfs(core_handle *spl, // set up the log if (spl->cfg.use_log) { - spl->log = log_create(cc, spl->cfg.log_cfg, spl->heap_id); + spl->log = shard_log_create( + cc, (shard_log_config *)spl->cfg.log_cfg, spl->heap_id); if (spl->log == NULL) { - platform_error_log("core_mkfs: log_create failed\n"); + platform_error_log("core_mkfs: shard_log_create failed\n"); rc = STATUS_NO_MEMORY; goto deinit_memtable_context; } @@ -2370,9 +2373,10 @@ core_mount(core_handle *spl, spl->mt_ctxt.rotate = core_rotate_log_virtual; if (spl->cfg.use_log) { - spl->log = log_create(cc, spl->cfg.log_cfg, spl->heap_id); + spl->log = shard_log_create( + cc, (shard_log_config *)spl->cfg.log_cfg, spl->heap_id); if (spl->log == NULL) { - platform_error_log("core_mount: log_create failed\n"); + platform_error_log("core_mount: shard_log_create failed\n"); rc = STATUS_NO_MEMORY; goto deinit_memtable_context; } @@ -2590,9 +2594,10 @@ core_checkpoint(core_handle *spl) rc = cache_durable_barrier(spl->cc); } if (SUCCESS(rc)) { - spl->log = log_create(spl->cc, spl->cfg.log_cfg, spl->heap_id); + spl->log = shard_log_create( + spl->cc, (shard_log_config *)spl->cfg.log_cfg, spl->heap_id); if (spl->log == NULL) { - platform_error_log("core_checkpoint: log_create failed\n"); + platform_error_log("core_checkpoint: shard_log_create failed\n"); rc = STATUS_NO_MEMORY; } } diff --git a/src/log.h b/src/log.h index 37fda13f..9daa461a 100644 --- a/src/log.h +++ b/src/log.h @@ -11,6 +11,7 @@ #include "cache.h" #include "data_internal.h" +#include "iterator.h" typedef struct log_handle log_handle; typedef struct log_iterator log_iterator; @@ -41,8 +42,8 @@ typedef int (*log_write_fn)(log_handle *log, * The caller must exclude concurrent log_write() and log_seal() calls. seal() * itself issues no writeback or durable barrier: to make the sealed pages * durable, the caller takes the cache writeback fence + a durable barrier - * afterward. The stream's identity is fixed at log_create() and obtained then - * via log_get_segment_info(), so seal needs no identity out-parameter; the + * afterward. The stream's identity is fixed at creation and obtained then via + * log_get_segment_info(), so seal needs no identity out-parameter; the * caller frees the on-disk extents later via log_dec_ref(). */ typedef platform_status (*log_seal_fn)(log_handle *log); @@ -95,8 +96,11 @@ log_get_segment_info(log_handle *log) return log->ops->segment_info(log); } -log_handle * -log_create(cache *cc, log_config *cfg, platform_heap_id hid); +/* + * A log_handle is created by the concrete log implementation -- e.g. + * shard_log_create() -- and then driven through the abstract ops above; it is + * freed by log_seal(). + */ /* * Release a sealed log segment identified by its log_segment_info: drop the @@ -106,3 +110,70 @@ log_create(cache *cc, log_config *cfg, platform_heap_id hid); */ void log_dec_ref(cache *cc, const log_segment_info *segment); + +/* + * ---- Abstract log iteration ---- + * + * A log_iterator reads a sealed log segment's records in generation order (used + * by crash recovery to replay a stream onto the durable root). It is a generic + * iterator (curr/can_next/next, via the embedded `super`) plus the log-specific + * ops below. To sub-class, make a log_iterator your first field. + */ +typedef void (*log_iterator_curr_generations_fn)(log_iterator *itor, + uint64 *memtable_generation, + uint64 *leaf_generation); +typedef void (*log_iterator_deinit_fn)(log_iterator *itor); + +typedef struct log_iterator_ops { + log_iterator_curr_generations_fn curr_generations; + log_iterator_deinit_fn deinit; +} log_iterator_ops; + +struct log_iterator { + iterator super; // generic iteration: curr / can_next / next + const log_iterator_ops *ops; +}; + +/* + * A log_iterator is created by the concrete log implementation -- e.g. + * shard_log_iterator_create() -- which fills in the ops below; callers then + * drive it through this abstract interface and free it with + * log_iterator_deinit(). + */ + +/* Whether a current record exists (safe to call curr / curr_generations). */ +static inline bool32 +log_iterator_can_next(log_iterator *itor) +{ + return iterator_can_next(&itor->super); +} + +/* The current record's key and message. Requires a current record. */ +static inline void +log_iterator_curr(log_iterator *itor, key *curr_key, message *msg) +{ + iterator_curr(&itor->super, curr_key, msg); +} + +/* The current record's generation metadata. Requires a current record. */ +static inline void +log_iterator_curr_generations(log_iterator *itor, + uint64 *memtable_generation, + uint64 *leaf_generation) +{ + itor->ops->curr_generations(itor, memtable_generation, leaf_generation); +} + +/* Advance to the next record. */ +static inline platform_status +log_iterator_next(log_iterator *itor) +{ + return iterator_next(&itor->super); +} + +/* Free the iterator and its resources; the handle is invalid afterward. */ +static inline void +log_iterator_deinit(log_iterator *itor) +{ + itor->ops->deinit(itor); +} diff --git a/src/shard_log.c b/src/shard_log.c index 40878537..4c7e35d6 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -60,6 +60,18 @@ const static iterator_ops shard_log_iterator_ops = { .print = NULL, }; +static void +shard_log_log_iterator_curr_generations(log_iterator *itor, + uint64 *memtable_generation, + uint64 *leaf_generation); +static void +shard_log_iterator_deinit(log_iterator *itor); + +const static log_iterator_ops shard_log_log_iterator_ops = { + .curr_generations = shard_log_log_iterator_curr_generations, + .deinit = shard_log_iterator_deinit, +}; + static const page_type shard_log_page_type_table[NUM_BLOB_BATCHES + 1] = { PAGE_TYPE_LOG, [1 ... NUM_BLOB_BATCHES] = PAGE_TYPE_BLOB, @@ -229,7 +241,6 @@ get_new_page_for_thread(shard_log *log, hdr->next_extent_addr = next_extent; hdr->num_entries = 0; thread_data->offset = sizeof(shard_log_hdr); - log->has_pages = TRUE; return 0; } @@ -478,17 +489,16 @@ shard_log_compare(const void *p1, const void *p2, void *unused) } log_handle * -log_create(cache *cc, log_config *lcfg, platform_heap_id hid) +shard_log_create(cache *cc, shard_log_config *cfg, platform_heap_id hid) { - shard_log_config *cfg = (shard_log_config *)lcfg; - shard_log *slog = TYPED_MALLOC(hid, slog); + shard_log *slog = TYPED_MALLOC(hid, slog); if (slog == NULL) { - platform_error_log("log_create: failed to allocate shard_log\n"); + platform_error_log("shard_log_create: failed to allocate shard_log\n"); return NULL; } platform_status rc = shard_log_init(slog, cc, cfg); if (!SUCCESS(rc)) { - platform_error_log("log_create: shard_log_init failed: %s\n", + platform_error_log("shard_log_create: shard_log_init failed: %s\n", platform_status_to_string(rc)); platform_free(hid, slog); return NULL; @@ -499,7 +509,7 @@ log_create(cache *cc, log_config *lcfg, platform_heap_id hid) return (log_handle *)slog; } -platform_status +static platform_status shard_log_iterator_init(cache *cc, shard_log_config *cfg, platform_heap_id hid, @@ -517,10 +527,12 @@ shard_log_iterator_init(cache *cc, uint64 contents_size; memset(itor, 0, sizeof(shard_log_iterator)); - itor->super.ops = &shard_log_iterator_ops; - itor->cc = cc; - itor->cfg = cfg; - allocator *al = cache_get_allocator(cc); + itor->super.super.ops = &shard_log_iterator_ops; // generic iterator + itor->super.ops = &shard_log_log_iterator_ops; // log_iterator + itor->heap_id = hid; + itor->cc = cc; + itor->cfg = cfg; + allocator *al = cache_get_allocator(cc); // traverse the log extents and calculate the required space extent_addr = addr; @@ -611,15 +623,42 @@ shard_log_iterator_init(cache *cc, return STATUS_OK; } -void -shard_log_iterator_deinit(platform_heap_id hid, shard_log_iterator *itor) +log_iterator * +shard_log_iterator_create(cache *cc, + shard_log_config *cfg, + platform_heap_id hid, + log_segment_info segment) { + shard_log_iterator *itor = TYPED_MALLOC(hid, itor); + if (itor == NULL) { + platform_error_log("shard_log_iterator_create: failed to allocate " + "shard_log_iterator\n"); + return NULL; + } + platform_status rc = + shard_log_iterator_init(cc, cfg, hid, segment.addr, segment.magic, itor); + if (!SUCCESS(rc)) { + platform_error_log("shard_log_iterator_create: shard_log_iterator_init " + "failed: %s\n", + platform_status_to_string(rc)); + platform_free(hid, itor); + return NULL; + } + return &itor->super; +} + +static void +shard_log_iterator_deinit(log_iterator *itorh) +{ + shard_log_iterator *itor = (shard_log_iterator *)itorh; + platform_heap_id hid = itor->heap_id; if (itor->contents != NULL) { platform_free(hid, itor->contents); } if (itor->entries != NULL) { platform_free(hid, itor->entries); } + platform_free(hid, itor); // the handle, from shard_log_iterator_create() } void @@ -630,11 +669,12 @@ shard_log_iterator_curr(iterator *itorh, key *curr_key, message *msg) *msg = log_entry_message(itor->cc, itor->entries[itor->pos]); } -void -shard_log_iterator_curr_generations(shard_log_iterator *itor, - uint64 *memtable_generation, - uint64 *leaf_generation) +static void +shard_log_log_iterator_curr_generations(log_iterator *itorh, + uint64 *memtable_generation, + uint64 *leaf_generation) { + shard_log_iterator *itor = (shard_log_iterator *)itorh; platform_assert(itor->pos < itor->num_entries); *memtable_generation = itor->entries[itor->pos]->memtable_generation; *leaf_generation = itor->entries[itor->pos]->leaf_generation; diff --git a/src/shard_log.h b/src/shard_log.h index d53e4f0f..c5f2649c 100644 --- a/src/shard_log.h +++ b/src/shard_log.h @@ -38,24 +38,22 @@ typedef struct shard_log_thread_data { * Sharded log context structure. */ typedef struct shard_log { - log_handle super; // handle to log I/O ops abstraction. - cache *cc; - shard_log_config *cfg; - platform_heap_id - heap_id; // heap the handle was allocated from; freed by seal + log_handle super; // handle to log I/O ops abstraction. + cache *cc; + shard_log_config *cfg; + platform_heap_id heap_id; shard_log_thread_data thread_data[MAX_THREADS]; mini_allocator mini; uint64 addr; uint64 meta_head; uint64 magic; - /* Set once any log page has been allocated; survives sealing. */ - bool32 has_pages; } shard_log; typedef struct log_entry log_entry; typedef struct shard_log_iterator { - iterator super; + log_iterator super; // IS-A log_iterator IS-A generic iterator + platform_heap_id heap_id; cache *cc; shard_log_config *cfg; char *contents; @@ -77,25 +75,25 @@ typedef struct ONDISK shard_log_hdr { uint16 num_entries; } shard_log_hdr; -platform_status -shard_log_iterator_init(cache *cc, - shard_log_config *cfg, - platform_heap_id hid, - uint64 addr, - uint64 magic, - shard_log_iterator *itor); - -void -shard_log_iterator_deinit(platform_heap_id hid, shard_log_iterator *itor); +/* + * Create a fresh sharded write-ahead log stream. Returns an abstract + * log_handle (or NULL on failure) to be driven through the log.h interface and + * retired with log_seal(). + */ +log_handle * +shard_log_create(cache *cc, shard_log_config *cfg, platform_heap_id hid); /* - * Return the generation metadata of the current record. The caller must - * first establish that the iterator has a current record. + * Create an iterator over the sharded log segment identified by `segment`, + * reading its records in generation order. Returns an abstract log_iterator + * (or NULL on failure) to be driven through the log.h interface and freed with + * log_iterator_deinit(). */ -void -shard_log_iterator_curr_generations(shard_log_iterator *itor, - uint64 *memtable_generation, - uint64 *leaf_generation); +log_iterator * +shard_log_iterator_create(cache *cc, + shard_log_config *cfg, + platform_heap_id hid, + log_segment_info segment); void shard_log_config_init(shard_log_config *log_cfg, diff --git a/tests/functional/log_test.c b/tests/functional/log_test.c index 034d67e8..984004ec 100644 --- a/tests/functional/log_test.c +++ b/tests/functional/log_test.c @@ -41,15 +41,14 @@ test_log_crash(clockcache *cc, key returned_key; message returned_message; log_segment_info segment; - shard_log_iterator itor; - iterator *itorh = (iterator *)&itor; + log_iterator *itor; char key_str[128]; char data_str[128]; merge_accumulator msg; DECLARE_AUTO_KEY_BUFFER(keybuffer, hid); platform_assert(cc != NULL); - logh = log_create((cache *)cc, (log_config *)cfg, hid); + logh = shard_log_create((cache *)cc, cfg, hid); platform_assert(logh != NULL); // The identity is fixed at creation; capture it before writing/sealing. @@ -103,21 +102,19 @@ test_log_crash(clockcache *cc, platform_assert_status_ok(rc); } - rc = shard_log_iterator_init( - (cache *)cc, cfg, hid, segment.addr, segment.magic, &itor); - platform_assert_status_ok(rc); - itorh = (iterator *)&itor; + itor = shard_log_iterator_create((cache *)cc, cfg, hid, segment); + platform_assert(itor != NULL); - for (i = 0; i < num_entries && iterator_can_curr(itorh); i++) { + for (i = 0; i < num_entries && log_iterator_can_next(itor); i++) { key skey = test_key(&keybuffer, TEST_RANDOM, i, 0, 0, 1 + (i % key_size), 0); generate_test_message(gen, i, &msg); message mmessage = merge_accumulator_to_message(&msg); - iterator_curr(itorh, &returned_key, &returned_message); + log_iterator_curr(itor, &returned_key, &returned_message); uint64 memtable_generation; uint64 leaf_generation; - shard_log_iterator_curr_generations( - &itor, &memtable_generation, &leaf_generation); + log_iterator_curr_generations( + itor, &memtable_generation, &leaf_generation); platform_assert(memtable_generation == i / LOG_TEST_LEAVES_PER_MEMTABLE); platform_assert(leaf_generation == i % LOG_TEST_LEAVES_PER_MEMTABLE); if (data_key_compare(cfg->data_cfg, skey, returned_key) @@ -132,17 +129,17 @@ test_log_crash(clockcache *cc, platform_default_log("actual: %s -- %s\n", key_str, data_str); platform_assert(0); } - rc = iterator_next(itorh); + rc = log_iterator_next(itor); platform_assert_status_ok(rc); } platform_default_log("log returned %lu of %lu entries\n", i, num_entries); platform_assert(i == num_entries); - platform_assert(!iterator_can_curr(itorh)); + platform_assert(!log_iterator_can_next(itor)); merge_accumulator_deinit(&msg); - shard_log_iterator_deinit(hid, &itor); + log_iterator_deinit(itor); log_dec_ref((cache *)cc, &segment); return 0; @@ -188,23 +185,21 @@ test_log_verify_segment(cache *cc, uint64 first_entry, uint64 num_entries) { - shard_log_iterator itor; - iterator *itorh = (iterator *)&itor; - merge_accumulator msg; + log_iterator *itor; + merge_accumulator msg; DECLARE_AUTO_KEY_BUFFER(keybuffer, hid); key returned_key; message returned_message; platform_assert(segment->addr != 0); platform_assert(segment->meta_addr != 0); - platform_status rc = shard_log_iterator_init( - cc, cfg, hid, segment->addr, segment->magic, &itor); - platform_assert_status_ok(rc); + itor = shard_log_iterator_create(cc, cfg, hid, *segment); + platform_assert(itor != NULL); merge_accumulator_init(&msg, hid); for (uint64 i = 0; i < num_entries; i++) { uint64 entry_num = first_entry + i; - platform_assert(iterator_can_curr(itorh)); + platform_assert(log_iterator_can_next(itor)); key skey = test_key(&keybuffer, TEST_RANDOM, entry_num, @@ -213,24 +208,24 @@ test_log_verify_segment(cache *cc, 1 + (entry_num % key_size), 0); generate_test_message(gen, entry_num, &msg); - iterator_curr(itorh, &returned_key, &returned_message); + log_iterator_curr(itor, &returned_key, &returned_message); uint64 memtable_generation; uint64 leaf_generation; - shard_log_iterator_curr_generations( - &itor, &memtable_generation, &leaf_generation); + log_iterator_curr_generations( + itor, &memtable_generation, &leaf_generation); platform_assert(memtable_generation == entry_num); platform_assert(leaf_generation == 0); platform_assert(data_key_compare(cfg->data_cfg, skey, returned_key) == 0); platform_assert( message_lex_cmp(merge_accumulator_to_message(&msg), returned_message) == 0); - rc = iterator_next(itorh); + platform_status rc = log_iterator_next(itor); platform_assert_status_ok(rc); } - platform_assert(!iterator_can_curr(itorh)); + platform_assert(!log_iterator_can_next(itor)); merge_accumulator_deinit(&msg); - shard_log_iterator_deinit(hid, &itor); + log_iterator_deinit(itor); } /* @@ -253,7 +248,7 @@ test_log_two_segments(clockcache *cc, const uint64 new_first = 2000, new_count = 16; log_segment_info sealed, fresh; - log_handle *log = log_create((cache *)cc, (log_config *)cfg, hid); + log_handle *log = shard_log_create((cache *)cc, cfg, hid); platform_assert(log != NULL); sealed = log_get_segment_info(log); // identity is fixed at creation test_log_write_range(log, gen, hid, key_size, old_first, old_count); @@ -276,7 +271,7 @@ test_log_two_segments(clockcache *cc, (cache *)cc, cfg, &sealed, gen, hid, key_size, old_first, old_count); // A fresh stream is a distinct segment: new mini allocator and new magic. - log = log_create((cache *)cc, (log_config *)cfg, hid); + log = shard_log_create((cache *)cc, cfg, hid); platform_assert(log != NULL); fresh = log_get_segment_info(log); test_log_write_range(log, gen, hid, key_size, new_first, new_count); @@ -311,8 +306,7 @@ test_log_large_message(cache *cc, shard_log_config *cfg, platform_heap_id hid) { platform_status rc; log_segment_info sealed; - shard_log_iterator itor; - iterator *itorh = (iterator *)&itor; + log_iterator *itor; merge_accumulator msg; key returned_key; message returned_message; @@ -320,7 +314,7 @@ test_log_large_message(cache *cc, shard_log_config *cfg, platform_heap_id hid) key skey = key_create(FALSE, sizeof(key_data) - 1, key_data); uint64 value_len = 3 * cache_page_size(cc) + 123; - log_handle *logh = log_create(cc, (log_config *)cfg, hid); + log_handle *logh = shard_log_create(cc, cfg, hid); platform_assert(logh != NULL); sealed = log_get_segment_info(logh); // identity is fixed at creation @@ -354,17 +348,17 @@ test_log_large_message(cache *cc, shard_log_config *cfg, platform_heap_id hid) rc = cache_durable_barrier(cc); platform_assert_status_ok(rc); - rc = shard_log_iterator_init(cc, cfg, hid, sealed.addr, sealed.magic, &itor); - platform_assert_status_ok(rc); - platform_assert(iterator_can_curr(itorh)); + itor = shard_log_iterator_create(cc, cfg, hid, sealed); + platform_assert(itor != NULL); + platform_assert(log_iterator_can_next(itor)); - iterator_curr(itorh, &returned_key, &returned_message); + log_iterator_curr(itor, &returned_key, &returned_message); platform_assert(data_key_compare(cfg->data_cfg, skey, returned_key) == 0); platform_assert( message_lex_cmp(merge_accumulator_to_message(&msg), returned_message) == 0); - shard_log_iterator_deinit(hid, &itor); + log_iterator_deinit(itor); merge_accumulator_deinit(&msg); log_dec_ref(cc, &sealed); return 0; @@ -423,7 +417,7 @@ test_log_perf(cache *cc, uint64 start_time; platform_status ret; - log_handle *logh = log_create((cache *)cc, (log_config *)cfg, hid); + log_handle *logh = shard_log_create((cache *)cc, cfg, hid); platform_assert(logh != NULL); log_segment_info sealed = log_get_segment_info(logh); From 0adc509d6b408f44a371f1efd0ef80b31b80336d Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Tue, 28 Jul 2026 02:27:25 -0700 Subject: [PATCH 32/64] Cleanup log Signed-off-by: Rob Johnson --- src/shard_log.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/shard_log.c b/src/shard_log.c index 4c7e35d6..0b476669 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -61,14 +61,14 @@ const static iterator_ops shard_log_iterator_ops = { }; static void -shard_log_log_iterator_curr_generations(log_iterator *itor, - uint64 *memtable_generation, - uint64 *leaf_generation); +shard_log_iterator_curr_generations(log_iterator *itor, + uint64 *memtable_generation, + uint64 *leaf_generation); static void shard_log_iterator_deinit(log_iterator *itor); const static log_iterator_ops shard_log_log_iterator_ops = { - .curr_generations = shard_log_log_iterator_curr_generations, + .curr_generations = shard_log_iterator_curr_generations, .deinit = shard_log_iterator_deinit, }; @@ -624,7 +624,7 @@ shard_log_iterator_init(cache *cc, } log_iterator * -shard_log_iterator_create(cache *cc, +shard_log_iterator_create(cache *cc, shard_log_config *cfg, platform_heap_id hid, log_segment_info segment) @@ -670,9 +670,9 @@ shard_log_iterator_curr(iterator *itorh, key *curr_key, message *msg) } static void -shard_log_log_iterator_curr_generations(log_iterator *itorh, - uint64 *memtable_generation, - uint64 *leaf_generation) +shard_log_iterator_curr_generations(log_iterator *itorh, + uint64 *memtable_generation, + uint64 *leaf_generation) { shard_log_iterator *itor = (shard_log_iterator *)itorh; platform_assert(itor->pos < itor->num_entries); From 4ba8009dd47e2823af0a4d7dd73035cb0f70e657 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Tue, 28 Jul 2026 02:44:28 -0700 Subject: [PATCH 33/64] log cleanup Signed-off-by: Rob Johnson --- src/shard_log.c | 408 +++++++++++++++++++++++------------------------- 1 file changed, 193 insertions(+), 215 deletions(-) diff --git a/src/shard_log.c b/src/shard_log.c index 0b476669..cdec940a 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -25,53 +25,6 @@ static uint64 shard_log_magic_idx = 0; -int -shard_log_write(log_handle *log, - key tuple_key, - message msg, - uint64 memtable_generation, - uint64 leaf_generation); -platform_status -shard_log_seal(log_handle *log); -log_segment_info -shard_log_get_segment_info(log_handle *log); - -static log_ops shard_log_ops = { - .write = shard_log_write, - .seal = shard_log_seal, - .segment_info = shard_log_get_segment_info, -}; - -void -shard_log_iterator_curr(iterator *itor, key *curr_key, message *msg); -bool32 -shard_log_iterator_can_prev(iterator *itor); -bool32 -shard_log_iterator_can_next(iterator *itor); -platform_status -shard_log_iterator_next(iterator *itor); - - -const static iterator_ops shard_log_iterator_ops = { - .curr = shard_log_iterator_curr, - .can_prev = shard_log_iterator_can_prev, - .can_next = shard_log_iterator_can_next, - .next = shard_log_iterator_next, - .print = NULL, -}; - -static void -shard_log_iterator_curr_generations(log_iterator *itor, - uint64 *memtable_generation, - uint64 *leaf_generation); -static void -shard_log_iterator_deinit(log_iterator *itor); - -const static log_iterator_ops shard_log_log_iterator_ops = { - .curr_generations = shard_log_iterator_curr_generations, - .deinit = shard_log_iterator_deinit, -}; - static const page_type shard_log_page_type_table[NUM_BLOB_BATCHES + 1] = { PAGE_TYPE_LOG, [1 ... NUM_BLOB_BATCHES] = PAGE_TYPE_BLOB, @@ -112,41 +65,6 @@ shard_log_alloc(shard_log *log, uint64 *next_extent) return cache_alloc(log->cc, addr, PAGE_TYPE_LOG); } -static platform_status -shard_log_init(shard_log *log, cache *cc, shard_log_config *cfg) -{ - memset(log, 0, sizeof(shard_log)); - log->cc = cc; - log->cfg = cfg; - log->super.ops = &shard_log_ops; - - uint64 magic_idx = __sync_fetch_and_add(&shard_log_magic_idx, 1); - log->magic = platform_checksum64(&magic_idx, sizeof(uint64), cfg->seed); - - allocator *al = cache_get_allocator(cc); - platform_status rc = allocator_alloc(al, &log->meta_head, PAGE_TYPE_LOG); - platform_assert_status_ok(rc); - - for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { - shard_log_thread_data *thread_data = - shard_log_get_thread_data(log, thr_i); - thread_data->addr = SHARD_UNMAPPED; - thread_data->offset = 0; - } - - log->addr = mini_init_with_types(&log->mini, - cc, - log->meta_head, - 0, - NUM_BLOB_BATCHES + 1, - PAGE_TYPE_LOG, - shard_log_page_type_table); - // platform_default_log("addr: %lu meta_head: %lu\n", log->addr, - // log->meta_head); - - return STATUS_OK; -} - /* * ------------------------------------------------------------------------- * Header for a key/message pair stored in the sharded log: Disk-resident @@ -391,6 +309,12 @@ shard_log_seal(log_handle *logh) page_handle *page = cache_get(cc, addr, TRUE, PAGE_TYPE_LOG); uint64 wait = 1; while (!cache_try_claim(cc, page)) { + /* + * Even though the stream is quiescent (no concurrent writers/seals), + * the background cache evictor can transiently hold the claim on a + * cleaned log page before it drains our read-ref, so we still retry + * rather than assert. + */ cache_unget(cc, page); platform_sleep_ns(wait); wait = wait > 1024 ? wait : 2 * wait; @@ -467,25 +391,45 @@ shard_log_next_extent_addr(shard_log_config *cfg, page_handle *page) return hdr->next_extent_addr; } -int -shard_log_compare(const void *p1, const void *p2, void *unused) +static log_ops shard_log_ops = { + .write = shard_log_write, + .seal = shard_log_seal, + .segment_info = shard_log_get_segment_info, +}; + +static platform_status +shard_log_init(shard_log *log, cache *cc, shard_log_config *cfg) { - log_entry **le1 = (log_entry **)p1; - log_entry **le2 = (log_entry **)p2; + memset(log, 0, sizeof(shard_log)); + log->cc = cc; + log->cfg = cfg; + log->super.ops = &shard_log_ops; - if ((*le1)->memtable_generation < (*le2)->memtable_generation) { - return -1; - } - if ((*le1)->memtable_generation > (*le2)->memtable_generation) { - return 1; - } - if ((*le1)->leaf_generation < (*le2)->leaf_generation) { - return -1; - } - if ((*le1)->leaf_generation > (*le2)->leaf_generation) { - return 1; + uint64 magic_idx = __sync_fetch_and_add(&shard_log_magic_idx, 1); + log->magic = platform_checksum64(&magic_idx, sizeof(uint64), cfg->seed); + + allocator *al = cache_get_allocator(cc); + platform_status rc = allocator_alloc(al, &log->meta_head, PAGE_TYPE_LOG); + platform_assert_status_ok(rc); + + for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { + shard_log_thread_data *thread_data = + shard_log_get_thread_data(log, thr_i); + thread_data->addr = SHARD_UNMAPPED; + thread_data->offset = 0; } - return 0; + + log->addr = mini_init_with_types(&log->mini, + cc, + log->meta_head, + 0, + NUM_BLOB_BATCHES + 1, + PAGE_TYPE_LOG, + shard_log_page_type_table); + // platform_default_log("addr: %lu meta_head: %lu\n", log->addr, + // log->meta_head); + + return STATUS_OK; } log_handle * @@ -509,6 +453,157 @@ shard_log_create(cache *cc, shard_log_config *cfg, platform_heap_id hid) return (log_handle *)slog; } +int +shard_log_compare(const void *p1, const void *p2, void *unused) +{ + log_entry **le1 = (log_entry **)p1; + log_entry **le2 = (log_entry **)p2; + + if ((*le1)->memtable_generation < (*le2)->memtable_generation) { + return -1; + } + if ((*le1)->memtable_generation > (*le2)->memtable_generation) { + return 1; + } + if ((*le1)->leaf_generation < (*le2)->leaf_generation) { + return -1; + } + if ((*le1)->leaf_generation > (*le2)->leaf_generation) { + return 1; + } + return 0; +} + +void +shard_log_iterator_curr(iterator *itorh, key *curr_key, message *msg) +{ + shard_log_iterator *itor = (shard_log_iterator *)itorh; + *curr_key = log_entry_key(itor->entries[itor->pos]); + *msg = log_entry_message(itor->cc, itor->entries[itor->pos]); +} + +static void +shard_log_iterator_curr_generations(log_iterator *itorh, + uint64 *memtable_generation, + uint64 *leaf_generation) +{ + shard_log_iterator *itor = (shard_log_iterator *)itorh; + platform_assert(itor->pos < itor->num_entries); + *memtable_generation = itor->entries[itor->pos]->memtable_generation; + *leaf_generation = itor->entries[itor->pos]->leaf_generation; +} + +bool32 +shard_log_iterator_can_prev(iterator *itorh) +{ + shard_log_iterator *itor = (shard_log_iterator *)itorh; + return itor->pos >= 0; +} + +bool32 +shard_log_iterator_can_next(iterator *itorh) +{ + shard_log_iterator *itor = (shard_log_iterator *)itorh; + return itor->pos < itor->num_entries; +} + +platform_status +shard_log_iterator_next(iterator *itorh) +{ + shard_log_iterator *itor = (shard_log_iterator *)itorh; + itor->pos++; + return STATUS_OK; +} + +/* + *----------------------------------------------------------------------------- + * shard_log_config_init -- + * + * Initialize shard_log config values + *----------------------------------------------------------------------------- + */ +void +shard_log_config_init(shard_log_config *log_cfg, + cache_config *cache_cfg, + data_config *data_cfg) +{ + ZERO_CONTENTS(log_cfg); + log_cfg->cache_cfg = cache_cfg; + log_cfg->data_cfg = data_cfg; + log_cfg->seed = HASH_SEED; + log_cfg->blob_cfg = (blob_build_config){ + .extent_batch = 1, + .page_batch = 2, + .subpage_batch = 3, + .alignment = cache_config_page_size(cache_cfg), + }; +} + +void +shard_log_print(shard_log *log) +{ + cache *cc = log->cc; + uint64 extent_addr = log->addr; + shard_log_config *cfg = log->cfg; + uint64 magic = log->magic; + data_config *dcfg = cfg->data_cfg; + uint64 pages_per_extent = shard_log_pages_per_extent(cfg); + allocator *al = cache_get_allocator(cc); + + while (extent_addr != 0 && allocator_get_refcount(al, extent_addr) > 0) { + cache_prefetch(cc, extent_addr, PAGE_TYPE_LOG); + uint64 next_extent_addr = 0; + for (uint64 i = 0; i < pages_per_extent; i++) { + uint64 page_addr = extent_addr + i * shard_log_page_size(cfg); + page_handle *page = cache_get(cc, page_addr, TRUE, PAGE_TYPE_LOG); + if (shard_log_valid(cfg, page, magic)) { + next_extent_addr = shard_log_next_extent_addr(cfg, page); + for (log_entry *le = first_log_entry(page->data); + !terminal_log_entry(cfg, page->data, le); + le = log_entry_next(le)) + { + platform_default_log( + "%s -- %s%s : memtable=%lu leaf=%lu\n", + key_string(dcfg, log_entry_key(le)), + log_entry_message_is_blob(le) ? "(blob) " : "", + message_string(dcfg, log_entry_message(cc, le)), + le->memtable_generation, + le->leaf_generation); + } + } + cache_unget(cc, page); + } + extent_addr = next_extent_addr; + } +} + +static void +shard_log_iterator_deinit(log_iterator *itorh) +{ + shard_log_iterator *itor = (shard_log_iterator *)itorh; + platform_heap_id hid = itor->heap_id; + if (itor->contents != NULL) { + platform_free(hid, itor->contents); + } + if (itor->entries != NULL) { + platform_free(hid, itor->entries); + } + platform_free(hid, itor); // the handle, from shard_log_iterator_create() +} + +const static iterator_ops shard_log_iterator_ops = { + .curr = shard_log_iterator_curr, + .can_prev = shard_log_iterator_can_prev, + .can_next = shard_log_iterator_can_next, + .next = shard_log_iterator_next, + .print = NULL, +}; + +const static log_iterator_ops shard_log_log_iterator_ops = { + .curr_generations = shard_log_iterator_curr_generations, + .deinit = shard_log_iterator_deinit, +}; + static platform_status shard_log_iterator_init(cache *cc, shard_log_config *cfg, @@ -646,120 +741,3 @@ shard_log_iterator_create(cache *cc, } return &itor->super; } - -static void -shard_log_iterator_deinit(log_iterator *itorh) -{ - shard_log_iterator *itor = (shard_log_iterator *)itorh; - platform_heap_id hid = itor->heap_id; - if (itor->contents != NULL) { - platform_free(hid, itor->contents); - } - if (itor->entries != NULL) { - platform_free(hid, itor->entries); - } - platform_free(hid, itor); // the handle, from shard_log_iterator_create() -} - -void -shard_log_iterator_curr(iterator *itorh, key *curr_key, message *msg) -{ - shard_log_iterator *itor = (shard_log_iterator *)itorh; - *curr_key = log_entry_key(itor->entries[itor->pos]); - *msg = log_entry_message(itor->cc, itor->entries[itor->pos]); -} - -static void -shard_log_iterator_curr_generations(log_iterator *itorh, - uint64 *memtable_generation, - uint64 *leaf_generation) -{ - shard_log_iterator *itor = (shard_log_iterator *)itorh; - platform_assert(itor->pos < itor->num_entries); - *memtable_generation = itor->entries[itor->pos]->memtable_generation; - *leaf_generation = itor->entries[itor->pos]->leaf_generation; -} - -bool32 -shard_log_iterator_can_prev(iterator *itorh) -{ - shard_log_iterator *itor = (shard_log_iterator *)itorh; - return itor->pos >= 0; -} - -bool32 -shard_log_iterator_can_next(iterator *itorh) -{ - shard_log_iterator *itor = (shard_log_iterator *)itorh; - return itor->pos < itor->num_entries; -} - -platform_status -shard_log_iterator_next(iterator *itorh) -{ - shard_log_iterator *itor = (shard_log_iterator *)itorh; - itor->pos++; - return STATUS_OK; -} - -/* - *----------------------------------------------------------------------------- - * shard_log_config_init -- - * - * Initialize shard_log config values - *----------------------------------------------------------------------------- - */ -void -shard_log_config_init(shard_log_config *log_cfg, - cache_config *cache_cfg, - data_config *data_cfg) -{ - ZERO_CONTENTS(log_cfg); - log_cfg->cache_cfg = cache_cfg; - log_cfg->data_cfg = data_cfg; - log_cfg->seed = HASH_SEED; - log_cfg->blob_cfg = (blob_build_config){ - .extent_batch = 1, - .page_batch = 2, - .subpage_batch = 3, - .alignment = cache_config_page_size(cache_cfg), - }; -} - -void -shard_log_print(shard_log *log) -{ - cache *cc = log->cc; - uint64 extent_addr = log->addr; - shard_log_config *cfg = log->cfg; - uint64 magic = log->magic; - data_config *dcfg = cfg->data_cfg; - uint64 pages_per_extent = shard_log_pages_per_extent(cfg); - allocator *al = cache_get_allocator(cc); - - while (extent_addr != 0 && allocator_get_refcount(al, extent_addr) > 0) { - cache_prefetch(cc, extent_addr, PAGE_TYPE_LOG); - uint64 next_extent_addr = 0; - for (uint64 i = 0; i < pages_per_extent; i++) { - uint64 page_addr = extent_addr + i * shard_log_page_size(cfg); - page_handle *page = cache_get(cc, page_addr, TRUE, PAGE_TYPE_LOG); - if (shard_log_valid(cfg, page, magic)) { - next_extent_addr = shard_log_next_extent_addr(cfg, page); - for (log_entry *le = first_log_entry(page->data); - !terminal_log_entry(cfg, page->data, le); - le = log_entry_next(le)) - { - platform_default_log( - "%s -- %s%s : memtable=%lu leaf=%lu\n", - key_string(dcfg, log_entry_key(le)), - log_entry_message_is_blob(le) ? "(blob) " : "", - message_string(dcfg, log_entry_message(cc, le)), - le->memtable_generation, - le->leaf_generation); - } - } - cache_unget(cc, page); - } - extent_addr = next_extent_addr; - } -} From dcfce5bb56e14db567c9fc7fab8050fa30ce2100 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Tue, 28 Jul 2026 03:09:12 -0700 Subject: [PATCH 34/64] log cleanup Signed-off-by: Rob Johnson --- src/core.c | 64 ++++++++++++++++++------------------ src/core.h | 4 +-- src/log.h | 62 +++++++++++++++++----------------- src/shard_log.c | 20 +++++------ src/shard_log.h | 10 +++--- src/superblock.c | 6 ++-- src/superblock.h | 18 +++++----- tests/functional/log_test.c | 18 +++++----- tests/unit/superblock_test.c | 8 ++--- 9 files changed, 105 insertions(+), 105 deletions(-) diff --git a/src/core.c b/src/core.c index b99c71c7..500de191 100644 --- a/src/core.c +++ b/src/core.c @@ -172,13 +172,13 @@ core_capture_checkpoint_cut(core_handle *spl, } /* - * Translate between the log module's log_segment_info and the superblock's - * (layout-identical but module-independent) superblock_log_info. + * Translate between the log module's log_head and the superblock's + * (layout-identical but module-independent) superblock_log_head. */ -static superblock_log_info -core_log_to_superblock(log_segment_info info) +static superblock_log_head +core_log_to_superblock(log_head info) { - return (superblock_log_info){ + return (superblock_log_head){ .addr = info.addr, .meta_addr = info.meta_addr, .magic = info.magic}; } @@ -194,7 +194,7 @@ core_log_to_superblock(log_segment_info info) * transferred to the durable record on success. */ static platform_status -core_publish_root_record(core_handle *spl, superblock_log_info live_log) +core_publish_root_record(core_handle *spl, superblock_log_head live_log) { platform_status rc; trunk_snapshot snapshot; @@ -371,12 +371,12 @@ core_checkpoint_maybe_begin(core_handle *spl) "core_checkpoint_maybe_begin: shard_log_create failed; skipping\n"); return; } - log_segment_info next_info = log_get_segment_info(next); + log_head next_head = log_get_head(next); platform_mutex_lock(&spl->checkpoint_state_lock); if (spl->checkpoint.phase == CORE_CHECKPOINT_IDLE) { spl->checkpoint.pending_log = next; - spl->checkpoint.live_info = next_info; + spl->checkpoint.live_head = next_head; spl->checkpoint.phase = CORE_CHECKPOINT_PENDING; spl->last_checkpoint_generation = spl->mt_ctxt.generation; next = NULL; // handed off to the checkpoint @@ -386,7 +386,7 @@ core_checkpoint_maybe_begin(core_handle *spl) if (next != NULL) { // Lost a race with a concurrent rotation; discard the speculative log. log_seal(next); - log_dec_ref(spl->cc, &next_info); + log_dec_ref(spl->cc, &next_head); } } @@ -405,7 +405,7 @@ core_rotate_log_virtual(void *arg, uint64 finalized_generation) platform_mutex_lock(&spl->checkpoint_state_lock); if (spl->checkpoint.phase == CORE_CHECKPOINT_PENDING) { spl->checkpoint.log_to_seal = spl->log; - spl->checkpoint.sealed_info = log_get_segment_info(spl->log); + spl->checkpoint.sealed_head = log_get_head(spl->log); spl->log = spl->checkpoint.pending_log; spl->checkpoint.pending_log = NULL; spl->checkpoint.cut_generation = finalized_generation; @@ -451,11 +451,11 @@ core_maybe_complete_checkpoint(core_handle *spl) bool32 complete = spl->checkpoint.phase == CORE_CHECKPOINT_INCORPORATING && retired != SUPERBLOCK_NO_INCORPORATED_GENERATION && retired >= spl->checkpoint.cut_generation; - log_segment_info sealed = {0}; - log_segment_info live = {0}; + log_head sealed = {0}; + log_head live = {0}; if (complete) { - sealed = spl->checkpoint.sealed_info; - live = spl->checkpoint.live_info; + sealed = spl->checkpoint.sealed_head; + live = spl->checkpoint.live_head; spl->checkpoint.phase = CORE_CHECKPOINT_COMPLETING; } platform_mutex_unlock(&spl->checkpoint_state_lock); @@ -476,8 +476,8 @@ core_maybe_complete_checkpoint(core_handle *spl) platform_mutex_lock(&spl->checkpoint_state_lock); if (SUCCESS(rc)) { - ZERO_CONTENTS(&spl->checkpoint.sealed_info); - ZERO_CONTENTS(&spl->checkpoint.live_info); + ZERO_CONTENTS(&spl->checkpoint.sealed_head); + ZERO_CONTENTS(&spl->checkpoint.live_head); spl->checkpoint.cut_generation = 0; spl->checkpoint.phase = CORE_CHECKPOINT_IDLE; } else { @@ -505,14 +505,14 @@ core_finish_checkpoint_for_shutdown(core_handle *spl) case CORE_CHECKPOINT_PENDING: // The next live log was pre-created but never installed; discard it. log_seal(cp->pending_log); - log_dec_ref(spl->cc, &cp->live_info); + log_dec_ref(spl->cc, &cp->live_head); break; case CORE_CHECKPOINT_INCORPORATING: case CORE_CHECKPOINT_COMPLETING: // The sealed log is fully incorporated after quiesce. The shutdown // publish records sealed=none, so just reclaim its extents here // (before the map is persisted, so the map reflects the free). - log_dec_ref(spl->cc, &cp->sealed_info); + log_dec_ref(spl->cc, &cp->sealed_head); break; case CORE_CHECKPOINT_SEALING: default: @@ -2241,9 +2241,9 @@ core_mkfs(core_handle *spl, // Establish the initial (empty) tree record, recording the live log; publish // it durably. No sealed log at mkfs. - superblock_log_info live_log = {0}; + superblock_log_head live_log = {0}; if (spl->cfg.use_log) { - live_log = core_log_to_superblock(log_get_segment_info(spl->log)); + live_log = core_log_to_superblock(log_get_head(spl->log)); } rc = core_publish_root_record(spl, live_log); if (!SUCCESS(rc)) { @@ -2420,8 +2420,8 @@ core_mount(core_handle *spl, */ superblock_log_cut( &spl->superblock, - spl->cfg.use_log ? core_log_to_superblock(log_get_segment_info(spl->log)) - : (superblock_log_info){0}); + spl->cfg.use_log ? core_log_to_superblock(log_get_head(spl->log)) + : (superblock_log_head){0}); rc = superblock_make_durable(&spl->superblock); if (!SUCCESS(rc)) { platform_error_log("core_mount: mark-dirty superblock_make_durable " @@ -2542,13 +2542,13 @@ core_teardown_after_shutdown(core_handle *spl) * durable root, so the live log is fully incorporated and discardable. Returns * an empty descriptor when logging is disabled. */ -static log_segment_info +static log_head core_seal_live_log(core_handle *spl) { if (!spl->cfg.use_log || spl->log == NULL) { - return (log_segment_info){0}; + return (log_head){0}; } - log_segment_info info = log_get_segment_info(spl->log); + log_head info = log_get_head(spl->log); log_seal(spl->log); spl->log = NULL; return info; @@ -2572,8 +2572,8 @@ platform_status core_checkpoint(core_handle *spl) { platform_status rc; - superblock_log_info new_live = {0}; - log_segment_info sealed = {0}; + superblock_log_head new_live = {0}; + log_head sealed = {0}; if (spl->cfg.use_log) { // --- Begin: seal the live log, start a fresh one, and publish @@ -2583,7 +2583,7 @@ core_checkpoint(core_handle *spl) return rc; } - sealed = log_get_segment_info(spl->log); + sealed = log_get_head(spl->log); log_seal(spl->log); // finalizes the pages (dirty), frees the handle spl->log = NULL; @@ -2604,7 +2604,7 @@ core_checkpoint(core_handle *spl) if (SUCCESS(rc)) { // Cut the log: the just-sealed live log becomes the sealed slot and // the fresh log becomes live. Root unchanged until completion. - new_live = core_log_to_superblock(log_get_segment_info(spl->log)); + new_live = core_log_to_superblock(log_get_head(spl->log)); superblock_log_cut(&spl->superblock, new_live); rc = superblock_make_durable(&spl->superblock); } @@ -2667,14 +2667,14 @@ core_unmount(core_handle *spl) * folded and discardable. Seal it (frees the handle) now; free its extents * after the cache flush below. */ - log_segment_info live_log = core_seal_live_log(spl); + log_head live_log = core_seal_live_log(spl); /* * Part A: publish the clean-unmount root with both log slots cleared (no * live or sealed log at rest). Allocation state stays invalid here; it * becomes valid only in Part B, after the map is persisted. */ - rc = core_publish_root_record(spl, (superblock_log_info){0}); + rc = core_publish_root_record(spl, (superblock_log_head){0}); if (!SUCCESS(rc)) { platform_error_log("core_unmount: failed to publish unmount root: %s\n", platform_status_to_string(rc)); @@ -2752,7 +2752,7 @@ core_destroy(core_handle *spl) // Discard the live log too: seal (frees the handle), then free its extents // after the cache flush. - log_segment_info live_log = core_seal_live_log(spl); + log_head live_log = core_seal_live_log(spl); core_teardown_after_shutdown(spl); log_dec_ref(spl->cc, &live_log); trunk_context_deinit(&spl->trunk_context); diff --git a/src/core.h b/src/core.h index 7dc76394..8fbf596b 100644 --- a/src/core.h +++ b/src/core.h @@ -129,8 +129,8 @@ typedef struct core_checkpoint_state { core_checkpoint_phase phase; log_handle *pending_log; // next live log, pre-created (PENDING) log_handle *log_to_seal; // old live log awaiting seal (SEALING) - log_segment_info sealed_info; // identity of the sealed log (reclaim) - log_segment_info live_info; // identity of the new live log + log_head sealed_head; // identity of the sealed log (reclaim) + log_head live_head; // identity of the new live log uint64 cut_generation; // complete once retired >= this } core_checkpoint_state; diff --git a/src/log.h b/src/log.h index 9daa461a..8e712881 100644 --- a/src/log.h +++ b/src/log.h @@ -18,16 +18,17 @@ typedef struct log_iterator log_iterator; typedef struct log_config log_config; /* - * Identity of one mini-allocator-backed log stream. It is sufficient for a - * higher-level checkpoint record to describe a stream, but not by itself a - * durable descriptor: core will later persist this information in an - * independently checksummed log-segment record. + * The on-disk head of one mini-allocator-backed log stream: the data head + * (where replay begins), the metadata head (which owns the stream's extents), + * and a per-stream magic that validates its pages. Fixed at creation; a + * higher-level checkpoint record stores it to later find the stream for replay + * or reclaim it via log_dec_ref(). */ -typedef struct log_segment_info { - uint64 addr; - uint64 meta_addr; - uint64 magic; -} log_segment_info; +typedef struct log_head { + uint64 addr; // data head: first log page, where replay begins + uint64 meta_addr; // mini-allocator metadata head; owns the stream's extents + uint64 magic; // per-stream magic; validates the stream's pages +} log_head; typedef int (*log_write_fn)(log_handle *log, key tuple_key, @@ -42,22 +43,22 @@ typedef int (*log_write_fn)(log_handle *log, * The caller must exclude concurrent log_write() and log_seal() calls. seal() * itself issues no writeback or durable barrier: to make the sealed pages * durable, the caller takes the cache writeback fence + a durable barrier - * afterward. The stream's identity is fixed at creation and obtained then via - * log_get_segment_info(), so seal needs no identity out-parameter; the - * caller frees the on-disk extents later via log_dec_ref(). + * afterward. The stream's head is fixed at creation and obtained then via + * log_get_head(), so seal needs no out-parameter; the caller frees the on-disk + * extents later via log_dec_ref(). */ typedef platform_status (*log_seal_fn)(log_handle *log); /* - * The stream's durable identity, fixed at creation. The caller records it + * The stream's durable head, fixed at creation. The caller records it * (e.g. in the superblock) as soon as the log is created, so that a crash * mid-stream can find the stream for replay. */ -typedef log_segment_info (*log_segment_info_fn)(log_handle *log); +typedef log_head (*log_head_fn)(log_handle *log); typedef struct log_ops { - log_write_fn write; - log_seal_fn seal; - log_segment_info_fn segment_info; + log_write_fn write; + log_seal_fn seal; + log_head_fn head; } log_ops; // to sub-class log, make a log_handle your first field @@ -79,8 +80,8 @@ log_write(log_handle *log, /* * Finalize and retire the log, freeing the handle. See log_seal_fn for the * required exclusion and durability ordering; the handle is invalid after this - * returns. Capture the identity via log_get_segment_info() beforehand (it is - * fixed at creation). + * returns. Capture the head via log_get_head() beforehand (it is fixed at + * creation). */ static inline platform_status log_seal(log_handle *log) @@ -88,12 +89,11 @@ log_seal(log_handle *log) return log->ops->seal(log); } -/* The stream's durable identity (fixed at creation). See log_segment_info_fn. - */ -static inline log_segment_info -log_get_segment_info(log_handle *log) +/* The stream's durable head (fixed at creation). See log_head_fn. */ +static inline log_head +log_get_head(log_handle *log) { - return log->ops->segment_info(log); + return log->ops->head(log); } /* @@ -103,19 +103,19 @@ log_get_segment_info(log_handle *log) */ /* - * Release a sealed log segment identified by its log_segment_info: drop the - * reference its metadata extent holds, freeing the segment's on-disk extents. - * Takes no handle -- the handle was freed by log_seal(); the caller retained - * only the identity (log_get_segment_info(), captured at creation). + * Release a sealed log identified by its log_head: drop the reference its + * metadata head holds, freeing the stream's on-disk extents. Takes no handle + * -- the handle was freed by log_seal(); the caller retained only the head + * (log_get_head(), captured at creation). */ void -log_dec_ref(cache *cc, const log_segment_info *segment); +log_dec_ref(cache *cc, const log_head *head); /* * ---- Abstract log iteration ---- * - * A log_iterator reads a sealed log segment's records in generation order (used - * by crash recovery to replay a stream onto the durable root). It is a generic + * A log_iterator reads a sealed log's records in generation order (used by + * crash recovery to replay a stream onto the durable root). It is a generic * iterator (curr/can_next/next, via the embedded `super`) plus the log-specific * ops below. To sub-class, make a log_iterator your first field. */ diff --git a/src/shard_log.c b/src/shard_log.c index cdec940a..58a518fe 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -284,7 +284,7 @@ shard_log_write(log_handle *logh, * make each page readable by shard_log_iterator_init(). Then it releases * the mini-allocator's unused reserve and frees the handle. After seal the * handle is invalid; the caller retains the identity it captured earlier - * (shard_log_get_segment_info(), fixed at creation) to reopen the stream + * (shard_log_get_head(), fixed at creation) to reopen the stream * for replay and, eventually, to free its extents via log_dec_ref(). * * The caller must prevent concurrent shard_log_write() and seal calls. @@ -355,7 +355,7 @@ shard_log_seal(log_handle *logh) } void -log_dec_ref(cache *cc, const log_segment_info *segment) +log_dec_ref(cache *cc, const log_head *segment) { if (segment->meta_addr == 0) { return; @@ -364,11 +364,11 @@ log_dec_ref(cache *cc, const log_segment_info *segment) platform_assert(ref == 0); } -log_segment_info -shard_log_get_segment_info(log_handle *logh) +log_head +shard_log_get_head(log_handle *logh) { shard_log *log = (shard_log *)logh; - return (log_segment_info){ + return (log_head){ .addr = log->addr, .meta_addr = log->meta_head, .magic = log->magic, @@ -392,9 +392,9 @@ shard_log_next_extent_addr(shard_log_config *cfg, page_handle *page) } static log_ops shard_log_ops = { - .write = shard_log_write, - .seal = shard_log_seal, - .segment_info = shard_log_get_segment_info, + .write = shard_log_write, + .seal = shard_log_seal, + .head = shard_log_get_head, }; static platform_status @@ -722,7 +722,7 @@ log_iterator * shard_log_iterator_create(cache *cc, shard_log_config *cfg, platform_heap_id hid, - log_segment_info segment) + log_head head) { shard_log_iterator *itor = TYPED_MALLOC(hid, itor); if (itor == NULL) { @@ -731,7 +731,7 @@ shard_log_iterator_create(cache *cc, return NULL; } platform_status rc = - shard_log_iterator_init(cc, cfg, hid, segment.addr, segment.magic, itor); + shard_log_iterator_init(cc, cfg, hid, head.addr, head.magic, itor); if (!SUCCESS(rc)) { platform_error_log("shard_log_iterator_create: shard_log_iterator_init " "failed: %s\n", diff --git a/src/shard_log.h b/src/shard_log.h index c5f2649c..9bb1a5ef 100644 --- a/src/shard_log.h +++ b/src/shard_log.h @@ -84,16 +84,16 @@ log_handle * shard_log_create(cache *cc, shard_log_config *cfg, platform_heap_id hid); /* - * Create an iterator over the sharded log segment identified by `segment`, - * reading its records in generation order. Returns an abstract log_iterator - * (or NULL on failure) to be driven through the log.h interface and freed with + * Create an iterator over the sharded log identified by `head`, reading its + * records in generation order. Returns an abstract log_iterator (or NULL on + * failure) to be driven through the log.h interface and freed with * log_iterator_deinit(). */ log_iterator * -shard_log_iterator_create(cache *cc, +shard_log_iterator_create(cache *cc, shard_log_config *cfg, platform_heap_id hid, - log_segment_info segment); + log_head head); void shard_log_config_init(shard_log_config *log_cfg, diff --git a/src/superblock.c b/src/superblock.c index b7a925b8..febf3d32 100644 --- a/src/superblock.c +++ b/src/superblock.c @@ -218,7 +218,7 @@ superblock_get_tree_record(const superblock_context *ctx, } void -superblock_log_cut(superblock_context *ctx, superblock_log_info new_live) +superblock_log_cut(superblock_context *ctx, superblock_log_head new_live) { // The current live log becomes the sealed log (its entries are being folded // into the next root); new_live receives subsequent inserts. The persisted @@ -232,7 +232,7 @@ void superblock_snapshot_tree(superblock_context *ctx, uint64 root_addr, uint64 incorporated_generation, - superblock_log_info new_live) + superblock_log_head new_live) { // The durable tree now includes everything folded into root_addr, so the // sealed log (if any) is done with; new_live is the log carried forward @@ -240,7 +240,7 @@ superblock_snapshot_tree(superblock_context *ctx, // allocation map, so invalidate it. ctx->image->tree.root_addr = root_addr; ctx->image->tree.incorporated_generation = incorporated_generation; - ctx->image->tree.sealed_log = (superblock_log_info){0}; + ctx->image->tree.sealed_log = (superblock_log_head){0}; ctx->image->tree.live_log = new_live; ctx->image->allocation_state_addr = 0; } diff --git a/src/superblock.h b/src/superblock.h index 2474bd35..c4bbd5b2 100644 --- a/src/superblock.h +++ b/src/superblock.h @@ -45,15 +45,15 @@ #define SUPERBLOCK_NUM_SLOTS (2) /* - * A log's on-disk identity. Mirrors log_segment_info's layout; the superblock - * stores it opaquely and does not depend on the log module. meta_addr == 0 - * means "no log present". + * A log's on-disk head. Mirrors log_head's layout; the superblock stores it + * opaquely and does not depend on the log module. meta_addr == 0 means "no log + * present". */ -typedef struct ONDISK superblock_log_info { +typedef struct ONDISK superblock_log_head { uint64 addr; uint64 meta_addr; uint64 magic; -} superblock_log_info; +} superblock_log_head; /* An empty (absent) log slot: meta_addr == 0. */ #define SUPERBLOCK_NO_LOG(info) ((info).meta_addr == 0) @@ -84,8 +84,8 @@ typedef struct ONDISK superblock_tree_record { * replay-skip boundary. */ uint64 incorporated_generation; - superblock_log_info live_log; - superblock_log_info sealed_log; + superblock_log_head live_log; + superblock_log_head sealed_log; } superblock_tree_record; /* Sentinel for superblock_tree_record.incorporated_generation. */ @@ -188,7 +188,7 @@ superblock_format(superblock_context *ctx, const allocator_config *cfg); * the sealed slot stays empty). */ void -superblock_log_cut(superblock_context *ctx, superblock_log_info new_live); +superblock_log_cut(superblock_context *ctx, superblock_log_head new_live); /* * Advance the durable tree to root_addr (having incorporated up to @@ -201,7 +201,7 @@ void superblock_snapshot_tree(superblock_context *ctx, uint64 root_addr, uint64 incorporated_generation, - superblock_log_info new_live); + superblock_log_head new_live); /* * Record the persisted allocator refcount map at map_addr as trustworthy. This diff --git a/tests/functional/log_test.c b/tests/functional/log_test.c index 984004ec..d3699b3a 100644 --- a/tests/functional/log_test.c +++ b/tests/functional/log_test.c @@ -40,7 +40,7 @@ test_log_crash(clockcache *cc, uint64 i; key returned_key; message returned_message; - log_segment_info segment; + log_head segment; log_iterator *itor; char key_str[128]; char data_str[128]; @@ -52,7 +52,7 @@ test_log_crash(clockcache *cc, platform_assert(logh != NULL); // The identity is fixed at creation; capture it before writing/sealing. - segment = log_get_segment_info(logh); + segment = log_get_head(logh); merge_accumulator_init(&msg, hid); @@ -178,7 +178,7 @@ test_log_write_range(log_handle *logh, static void test_log_verify_segment(cache *cc, shard_log_config *cfg, - const log_segment_info *segment, + const log_head *segment, test_message_generator *gen, platform_heap_id hid, uint64 key_size, @@ -246,11 +246,11 @@ test_log_two_segments(clockcache *cc, { const uint64 old_first = 1000, old_count = 16; const uint64 new_first = 2000, new_count = 16; - log_segment_info sealed, fresh; + log_head sealed, fresh; log_handle *log = shard_log_create((cache *)cc, cfg, hid); platform_assert(log != NULL); - sealed = log_get_segment_info(log); // identity is fixed at creation + sealed = log_get_head(log); // identity is fixed at creation test_log_write_range(log, gen, hid, key_size, old_first, old_count); platform_status rc = log_seal(log); // frees log @@ -273,7 +273,7 @@ test_log_two_segments(clockcache *cc, // A fresh stream is a distinct segment: new mini allocator and new magic. log = shard_log_create((cache *)cc, cfg, hid); platform_assert(log != NULL); - fresh = log_get_segment_info(log); + fresh = log_get_head(log); test_log_write_range(log, gen, hid, key_size, new_first, new_count); rc = log_seal(log); // frees log platform_assert_status_ok(rc); @@ -305,7 +305,7 @@ static int test_log_large_message(cache *cc, shard_log_config *cfg, platform_heap_id hid) { platform_status rc; - log_segment_info sealed; + log_head sealed; log_iterator *itor; merge_accumulator msg; key returned_key; @@ -316,7 +316,7 @@ test_log_large_message(cache *cc, shard_log_config *cfg, platform_heap_id hid) log_handle *logh = shard_log_create(cc, cfg, hid); platform_assert(logh != NULL); - sealed = log_get_segment_info(logh); // identity is fixed at creation + sealed = log_get_head(logh); // identity is fixed at creation merge_accumulator_init(&msg, hid); bool32 success = merge_accumulator_resize(&msg, value_len); @@ -419,7 +419,7 @@ test_log_perf(cache *cc, log_handle *logh = shard_log_create((cache *)cc, cfg, hid); platform_assert(logh != NULL); - log_segment_info sealed = log_get_segment_info(logh); + log_head sealed = log_get_head(logh); for (uint64 i = 0; i < num_threads; i++) { params[i].logh = logh; diff --git a/tests/unit/superblock_test.c b/tests/unit/superblock_test.c index bb53d7da..792148cb 100644 --- a/tests/unit/superblock_test.c +++ b/tests/unit/superblock_test.c @@ -125,7 +125,7 @@ CTEST2(superblock, test_snapshot_persists_state) rc = superblock_format(&ctx, &data->allocator_cfg); ASSERT_TRUE(SUCCESS(rc)); - superblock_log_info live = { + superblock_log_head live = { .addr = 0x6000, .meta_addr = 0x8000, .magic = 0x11}; superblock_snapshot_tree( &ctx, 0x4000, SUPERBLOCK_NO_INCORPORATED_GENERATION, live); @@ -158,10 +158,10 @@ CTEST2(superblock, test_snapshot_persists_state) } /* Steady, begin-checkpoint, and complete-checkpoint tree-record states. */ -static const superblock_log_info TEST_LOG_L1 = {.addr = 0x6000, +static const superblock_log_head TEST_LOG_L1 = {.addr = 0x6000, .meta_addr = 0x8000, .magic = 0x11}; -static const superblock_log_info TEST_LOG_L2 = {.addr = 0x10000, +static const superblock_log_head TEST_LOG_L2 = {.addr = 0x10000, .meta_addr = 0x12000, .magic = 0x22}; @@ -281,7 +281,7 @@ CTEST2(superblock, test_torn_write_falls_back_to_older_generation) superblock_snapshot_tree(&ctx, 0x4000, SUPERBLOCK_NO_INCORPORATED_GENERATION, - (superblock_log_info){0}); + (superblock_log_head){0}); rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); superblock_context_deinit(&ctx); From ac06b14f09b42e7569ab99468033afe9442506c4 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Tue, 28 Jul 2026 03:22:05 -0700 Subject: [PATCH 35/64] memtable cleanup Signed-off-by: Rob Johnson --- src/memtable.c | 11 +++++++++-- src/memtable.h | 11 ----------- tests/functional/btree_test.c | 3 --- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/memtable.c b/src/memtable.c index 9b537b20..7a0d2310 100644 --- a/src/memtable.c +++ b/src/memtable.c @@ -80,7 +80,14 @@ memtable_end_insert(memtable_context *ctxt) batch_rwlock_unget(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); } -void +/* + * Exclude all inserts, including one that has already acquired its shared + * insert lock, so the caller can safely mutate generation state. Used by + * memtable_force_finalize(); the normal rotation path instead upgrades its own + * insert lock via memtable_try_begin_insert_rotation(). Pair with + * memtable_unblock_inserts(). + */ +static void memtable_block_inserts(memtable_context *ctxt) { batch_rwlock_get(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); @@ -88,7 +95,7 @@ memtable_block_inserts(memtable_context *ctxt) batch_rwlock_lock(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); } -void +static void memtable_unblock_inserts(memtable_context *ctxt) { batch_rwlock_full_unlock(&ctxt->rwlock, MEMTABLE_INSERT_LOCK_IDX); diff --git a/src/memtable.h b/src/memtable.h index f0351115..3ab8ef37 100644 --- a/src/memtable.h +++ b/src/memtable.h @@ -168,17 +168,6 @@ memtable_maybe_rotate_and_begin_insert(memtable_context *ctxt, void memtable_end_insert(memtable_context *ctxt); -/* - * Exclude all inserts, including an insert that has already acquired its - * shared insert lock. A checkpoint uses this around sealing the log prefix - * that describes its snapshot. Must be paired with memtable_unblock_inserts. - */ -void -memtable_block_inserts(memtable_context *ctxt); - -void -memtable_unblock_inserts(memtable_context *ctxt); - void memtable_begin_lookup(memtable_context *ctxt); diff --git a/tests/functional/btree_test.c b/tests/functional/btree_test.c index 560e4c24..18f0efeb 100644 --- a/tests/functional/btree_test.c +++ b/tests/functional/btree_test.c @@ -150,9 +150,6 @@ test_memtable_generation_init(cache *cc, } } - memtable_block_inserts(&mt_ctxt); - memtable_unblock_inserts(&mt_ctxt); - deinit_recovery: memtable_context_deinit(&mt_ctxt); if (SUCCESS(rc)) { From e51b09db20ce02214fec68cd08c8dc2406511d7a Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Tue, 28 Jul 2026 03:32:17 -0700 Subject: [PATCH 36/64] comment formatting cleanup Signed-off-by: Rob Johnson --- src/core.h | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/core.h b/src/core.h index 8fbf596b..ba91ca2a 100644 --- a/src/core.h +++ b/src/core.h @@ -105,10 +105,12 @@ typedef struct core_handle core_handle; * PENDING the next live log is pre-created; the next memtable rotation * will swap it in under the insert lock. * SEALING the rotation swapped the new live log in; the old log still - * needs sealing (done just after the rotation critical - * section). INCORPORATING the old log is sealed; waiting for its generations to - * be incorporated into the trunk root. COMPLETING the completion publish - * (advance root, clear sealed slot) is in flight. + * needs sealing (which will be performed just after the + * rotation critical section). + * INCORPORATING the old log is sealed; waiting for its generations to + * be incorporated into the trunk root. + * COMPLETING the completion publish (advance root, clear sealed slot) is + * in flight. * * The only transition that touches the shared spl->log pointer (PENDING -> * SEALING) runs inside the memtable rotation critical section, where the insert @@ -129,8 +131,8 @@ typedef struct core_checkpoint_state { core_checkpoint_phase phase; log_handle *pending_log; // next live log, pre-created (PENDING) log_handle *log_to_seal; // old live log awaiting seal (SEALING) - log_head sealed_head; // identity of the sealed log (reclaim) - log_head live_head; // identity of the new live log + log_head sealed_head; // identity of the sealed log (reclaim) + log_head live_head; // identity of the new live log uint64 cut_generation; // complete once retired >= this } core_checkpoint_state; From 24eae2d9d368ea7f2f7e5463acb5855c701e43a6 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Tue, 28 Jul 2026 03:37:43 -0700 Subject: [PATCH 37/64] core cleanup Signed-off-by: Rob Johnson --- src/core.c | 18 ++++++------------ src/core.h | 2 -- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/core.c b/src/core.c index 500de191..92669908 100644 --- a/src/core.c +++ b/src/core.c @@ -55,14 +55,14 @@ core_checkpoint_lock_init(core_handle *spl) if (!SUCCESS(rc)) { return rc; } - spl->checkpoint_lock_initialized = TRUE; rc = platform_mutex_init( &spl->checkpoint_state_lock, platform_get_module_id(), spl->heap_id); if (!SUCCESS(rc)) { + platform_status destroy_rc = platform_mutex_destroy(&spl->checkpoint_lock); + platform_assert_status_ok(destroy_rc); return rc; } - spl->checkpoint_state_lock_initialized = TRUE; ZERO_CONTENTS(&spl->checkpoint); // phase == CORE_CHECKPOINT_IDLE spl->last_checkpoint_generation = 0; @@ -72,16 +72,10 @@ core_checkpoint_lock_init(core_handle *spl) static void core_checkpoint_lock_deinit(core_handle *spl) { - if (spl->checkpoint_state_lock_initialized) { - platform_status rc = platform_mutex_destroy(&spl->checkpoint_state_lock); - platform_assert_status_ok(rc); - spl->checkpoint_state_lock_initialized = FALSE; - } - if (spl->checkpoint_lock_initialized) { - platform_status rc = platform_mutex_destroy(&spl->checkpoint_lock); - platform_assert_status_ok(rc); - spl->checkpoint_lock_initialized = FALSE; - } + platform_status rc = platform_mutex_destroy(&spl->checkpoint_state_lock); + platform_assert_status_ok(rc); + rc = platform_mutex_destroy(&spl->checkpoint_lock); + platform_assert_status_ok(rc); } /* diff --git a/src/core.h b/src/core.h index ba91ca2a..bc0f065d 100644 --- a/src/core.h +++ b/src/core.h @@ -173,7 +173,6 @@ struct core_handle { /* Serializes snapshot cuts and superblock publication. */ platform_mutex checkpoint_lock; - bool32 checkpoint_lock_initialized; /* * Incorporation-driven checkpoint state. checkpoint_state_lock guards the @@ -182,7 +181,6 @@ struct core_handle { * inside the memtable rotation critical section cannot stall inserts on I/O. */ platform_mutex checkpoint_state_lock; - bool32 checkpoint_state_lock_initialized; core_checkpoint_state checkpoint; uint64 last_checkpoint_generation; From eb05adbd93d85f29c830408da42bebabd5b7eb9b Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Tue, 28 Jul 2026 03:37:50 -0700 Subject: [PATCH 38/64] formatting Signed-off-by: Rob Johnson --- src/core.c | 13 ++++++----- tests/functional/log_test.c | 46 ++++++++++++++++++------------------- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/core.c b/src/core.c index 92669908..8115cce8 100644 --- a/src/core.c +++ b/src/core.c @@ -59,7 +59,8 @@ core_checkpoint_lock_init(core_handle *spl) rc = platform_mutex_init( &spl->checkpoint_state_lock, platform_get_module_id(), spl->heap_id); if (!SUCCESS(rc)) { - platform_status destroy_rc = platform_mutex_destroy(&spl->checkpoint_lock); + platform_status destroy_rc = + platform_mutex_destroy(&spl->checkpoint_lock); platform_assert_status_ok(destroy_rc); return rc; } @@ -2412,10 +2413,10 @@ core_mount(core_handle *spl, * silently reverting to this now-stale root. The root is unchanged; a clean * mount has no prior live log, so the sealed slot stays empty. */ - superblock_log_cut( - &spl->superblock, - spl->cfg.use_log ? core_log_to_superblock(log_get_head(spl->log)) - : (superblock_log_head){0}); + superblock_log_cut(&spl->superblock, + spl->cfg.use_log + ? core_log_to_superblock(log_get_head(spl->log)) + : (superblock_log_head){0}); rc = superblock_make_durable(&spl->superblock); if (!SUCCESS(rc)) { platform_error_log("core_mount: mark-dirty superblock_make_durable " @@ -2567,7 +2568,7 @@ core_checkpoint(core_handle *spl) { platform_status rc; superblock_log_head new_live = {0}; - log_head sealed = {0}; + log_head sealed = {0}; if (spl->cfg.use_log) { // --- Begin: seal the live log, start a fresh one, and publish diff --git a/tests/functional/log_test.c b/tests/functional/log_test.c index d3699b3a..2cdcb39d 100644 --- a/tests/functional/log_test.c +++ b/tests/functional/log_test.c @@ -35,16 +35,16 @@ test_log_crash(clockcache *cc, bool32 crash) { - platform_status rc; - log_handle *logh; - uint64 i; - key returned_key; - message returned_message; - log_head segment; - log_iterator *itor; - char key_str[128]; - char data_str[128]; - merge_accumulator msg; + platform_status rc; + log_handle *logh; + uint64 i; + key returned_key; + message returned_message; + log_head segment; + log_iterator *itor; + char key_str[128]; + char data_str[128]; + merge_accumulator msg; DECLARE_AUTO_KEY_BUFFER(keybuffer, hid); platform_assert(cc != NULL); @@ -178,7 +178,7 @@ test_log_write_range(log_handle *logh, static void test_log_verify_segment(cache *cc, shard_log_config *cfg, - const log_head *segment, + const log_head *segment, test_message_generator *gen, platform_heap_id hid, uint64 key_size, @@ -244,9 +244,9 @@ test_log_two_segments(clockcache *cc, test_message_generator *gen, uint64 key_size) { - const uint64 old_first = 1000, old_count = 16; - const uint64 new_first = 2000, new_count = 16; - log_head sealed, fresh; + const uint64 old_first = 1000, old_count = 16; + const uint64 new_first = 2000, new_count = 16; + log_head sealed, fresh; log_handle *log = shard_log_create((cache *)cc, cfg, hid); platform_assert(log != NULL); @@ -304,15 +304,15 @@ test_log_two_segments(clockcache *cc, static int test_log_large_message(cache *cc, shard_log_config *cfg, platform_heap_id hid) { - platform_status rc; - log_head sealed; - log_iterator *itor; - merge_accumulator msg; - key returned_key; - message returned_message; - char key_data[] = "large-log-key"; - key skey = key_create(FALSE, sizeof(key_data) - 1, key_data); - uint64 value_len = 3 * cache_page_size(cc) + 123; + platform_status rc; + log_head sealed; + log_iterator *itor; + merge_accumulator msg; + key returned_key; + message returned_message; + char key_data[] = "large-log-key"; + key skey = key_create(FALSE, sizeof(key_data) - 1, key_data); + uint64 value_len = 3 * cache_page_size(cc) + 123; log_handle *logh = shard_log_create(cc, cfg, hid); platform_assert(logh != NULL); From 8430b9826074fcacdb6f63b0b89c675ce3a13995 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Tue, 28 Jul 2026 11:47:09 -0700 Subject: [PATCH 39/64] simplify generation semantics in superblock Signed-off-by: Rob Johnson --- src/core.c | 42 +++++++++++++++++++----------------- src/superblock.c | 19 ++++++++-------- src/superblock.h | 31 ++++++++++++-------------- tests/unit/splinter_test.c | 7 +++--- tests/unit/superblock_test.c | 13 ++++------- 5 files changed, 54 insertions(+), 58 deletions(-) diff --git a/src/core.c b/src/core.c index 8115cce8..a911e622 100644 --- a/src/core.c +++ b/src/core.c @@ -143,8 +143,7 @@ core_close_log_stream_if_enabled(core_handle *spl, static platform_status core_capture_checkpoint_cut(core_handle *spl, trunk_snapshot *snapshot, - bool32 *has_incorporated_generation, - uint64 *incorporated_generation) + uint64 *first_unincorporated_generation) { /* * Incorporation publishes its generation and root while holding lookup @@ -160,9 +159,13 @@ core_capture_checkpoint_cut(core_handle *spl, return rc; } - *has_incorporated_generation = retired_generation != UINT64_MAX; - *incorporated_generation = - *has_incorporated_generation ? retired_generation : 0; + /* + * The first generation not folded into the root -- the exclusive replay + * bound. When nothing has been retired, memtable_generation_retired() is + * UINT64_MAX and this wraps to 0 ("replay from generation 0"), so no sentinel + * is needed. + */ + *first_unincorporated_generation = retired_generation + 1; return STATUS_OK; } @@ -193,8 +196,7 @@ core_publish_root_record(core_handle *spl, superblock_log_head live_log) { platform_status rc; trunk_snapshot snapshot; - bool32 has_incorporated_generation; - uint64 incorporated_generation; + uint64 first_unincorporated_generation; uint64 old_root_addr = 0; superblock_tree_record old_rec; @@ -208,7 +210,7 @@ core_publish_root_record(core_handle *spl, superblock_log_head live_log) } rc = core_capture_checkpoint_cut( - spl, &snapshot, &has_incorporated_generation, &incorporated_generation); + spl, &snapshot, &first_unincorporated_generation); if (!SUCCESS(rc)) { goto unlock_checkpoint; } @@ -241,9 +243,7 @@ core_publish_root_record(core_handle *spl, superblock_log_head live_log) */ superblock_snapshot_tree(&spl->superblock, snapshot.root_addr, - has_incorporated_generation - ? incorporated_generation - : SUPERBLOCK_NO_INCORPORATED_GENERATION, + first_unincorporated_generation, live_log); rc = superblock_make_durable(&spl->superblock); @@ -442,10 +442,13 @@ static platform_status core_maybe_complete_checkpoint(core_handle *spl) { platform_mutex_lock(&spl->checkpoint_state_lock); - uint64 retired = memtable_generation_retired(&spl->mt_ctxt); + // The sealed log's cut generation is fully incorporated once it falls below + // the first unincorporated generation. When nothing has been retired, + // memtable_generation_retired() is UINT64_MAX and this wraps to 0, so the + // comparison is false without a sentinel check. + uint64 first_unincorporated = memtable_generation_retired(&spl->mt_ctxt) + 1; bool32 complete = spl->checkpoint.phase == CORE_CHECKPOINT_INCORPORATING - && retired != SUPERBLOCK_NO_INCORPORATED_GENERATION - && retired >= spl->checkpoint.cut_generation; + && first_unincorporated > spl->checkpoint.cut_generation; log_head sealed = {0}; log_head live = {0}; if (complete) { @@ -2344,10 +2347,9 @@ core_mount(core_handle *spl, } uint64 root_addr = rec.root_addr; - uint64 resume_generation = - rec.incorporated_generation == SUPERBLOCK_NO_INCORPORATED_GENERATION - ? 0 - : rec.incorporated_generation + 1; + // The record already stores the first unincorporated generation, which is + // exactly where the memtable resumes (0 for a fresh, never-incorporated db). + uint64 resume_generation = rec.first_unincorporated_generation; memtable_config *mt_cfg = &spl->cfg.mt_cfg; rc = memtable_context_init_at_generation(&spl->mt_ctxt, @@ -2805,14 +2807,14 @@ core_print_super_block(platform_log_handle *log_handle, core_handle *spl) platform_log(log_handle, "Superblock tree record root_id=%lu {\n" - " root_addr=%lu incorporated_generation=%lu\n" + " root_addr=%lu first_unincorporated_generation=%lu\n" " live_log: meta_addr=%lu addr=%lu magic=%lu\n" " sealed_log: meta_addr=%lu addr=%lu magic=%lu\n" " allocation_state: %s (addr=%lu)\n" "}\n\n", spl->id, rec.root_addr, - rec.incorporated_generation, + rec.first_unincorporated_generation, rec.live_log.meta_addr, rec.live_log.addr, rec.live_log.magic, diff --git a/src/superblock.c b/src/superblock.c index febf3d32..6792ad66 100644 --- a/src/superblock.c +++ b/src/superblock.c @@ -180,9 +180,9 @@ superblock_format(superblock_context *ctx, const allocator_config *cfg) ctx->image->format_version = SUPERBLOCK_FORMAT_VERSION; ctx->image->generation = 0; ctx->image->allocation_state_addr = 0; // fresh DB: rebuild on crash - // A fresh, empty tree: no root and nothing incorporated yet. - ctx->image->tree.incorporated_generation = - SUPERBLOCK_NO_INCORPORATED_GENERATION; + // A fresh, empty tree: no root, and nothing incorporated yet -- replay from + // generation 0. + ctx->image->tree.first_unincorporated_generation = 0; /* * Write both physical copies so torn-write protection is in force from the @@ -231,18 +231,19 @@ superblock_log_cut(superblock_context *ctx, superblock_log_head new_live) void superblock_snapshot_tree(superblock_context *ctx, uint64 root_addr, - uint64 incorporated_generation, + uint64 first_unincorporated_generation, superblock_log_head new_live) { // The durable tree now includes everything folded into root_addr, so the // sealed log (if any) is done with; new_live is the log carried forward // (empty at a clean shutdown). Advancing the root diverges the persisted // allocation map, so invalidate it. - ctx->image->tree.root_addr = root_addr; - ctx->image->tree.incorporated_generation = incorporated_generation; - ctx->image->tree.sealed_log = (superblock_log_head){0}; - ctx->image->tree.live_log = new_live; - ctx->image->allocation_state_addr = 0; + ctx->image->tree.root_addr = root_addr; + ctx->image->tree.first_unincorporated_generation = + first_unincorporated_generation; + ctx->image->tree.sealed_log = (superblock_log_head){0}; + ctx->image->tree.live_log = new_live; + ctx->image->allocation_state_addr = 0; } void diff --git a/src/superblock.h b/src/superblock.h index c4bbd5b2..16a2540f 100644 --- a/src/superblock.h +++ b/src/superblock.h @@ -67,30 +67,27 @@ typedef struct ONDISK superblock_log_head { * checkpoint is in progress -- the just-sealed stream whose entries are being * folded into the new root. At rest (between checkpoints, and after a clean * unmount) sealed_log is empty. On crash recovery the sealed log (if present) - * then the live log are replayed onto root_addr, skipping entries at or below - * incorporated_generation. + * then the live log are replayed onto root_addr, replaying entries at or above + * first_unincorporated_generation. * * There is intentionally no clean/dirty ("unmounted") flag: replaying a clean - * root is a no-op because every entry is at or below incorporated_generation, + * root is a no-op because every entry is below first_unincorporated_generation, * so the superblock's allocation_state_addr validity is the single at-rest * signal. */ typedef struct ONDISK superblock_tree_record { uint64 root_addr; /* - * Highest memtable generation folded into root_addr; UINT64_MAX means none - * has been incorporated yet (distinct from generation 0, which is a real, - * live generation). Drives memtable-generation resume at mount and the - * replay-skip boundary. + * First memtable generation NOT folded into root_addr -- the exclusive + * replay bound and the generation to resume at on mount. 0 means nothing + * has been incorporated yet (replay from the start); no sentinel is needed + * because generation 0 being unincorporated is the natural fresh state. */ - uint64 incorporated_generation; + uint64 first_unincorporated_generation; superblock_log_head live_log; superblock_log_head sealed_log; } superblock_tree_record; -/* Sentinel for superblock_tree_record.incorporated_generation. */ -#define SUPERBLOCK_NO_INCORPORATED_GENERATION (UINT64_MAX) - typedef struct ONDISK superblock { disk_geometry geometry; // MUST be first; see the bootstrap-read note above. uint64 format_magic; @@ -191,16 +188,16 @@ void superblock_log_cut(superblock_context *ctx, superblock_log_head new_live); /* - * Advance the durable tree to root_addr (having incorporated up to - * incorporated_generation) and clear the sealed log -- a snapshot is taken only - * after the sealed log has been folded into the root. new_live is the log - * carried forward (empty at a clean shutdown). Invalidates the allocation - * state. Used at a checkpoint's completion and at a clean unmount. + * Advance the durable tree to root_addr (with first_unincorporated_generation + * the first generation not folded in) and clear the sealed log -- a snapshot is + * taken only after the sealed log has been folded into the root. new_live is + * the log carried forward (empty at a clean shutdown). Invalidates the + * allocation state. Used at a checkpoint's completion and at a clean unmount. */ void superblock_snapshot_tree(superblock_context *ctx, uint64 root_addr, - uint64 incorporated_generation, + uint64 first_unincorporated_generation, superblock_log_head new_live); /* diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index 3b481cc1..96f17054 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -360,9 +360,10 @@ run_auto_checkpoint_workload(void *datap, uint64 interval) superblock_tree_record rec; superblock_get_tree_record(&spl.superblock, &rec); ASSERT_TRUE(SUPERBLOCK_NO_LOG(rec.sealed_log)); - // A checkpoint published an advanced, incorporated durable root mid-run. - ASSERT_NOT_EQUAL(SUPERBLOCK_NO_INCORPORATED_GENERATION, - rec.incorporated_generation); + // A checkpoint published an advanced, incorporated durable root mid-run: + // at least one generation was folded in, so the first unincorporated + // generation has advanced past 0. + ASSERT_NOT_EQUAL(0, rec.first_unincorporated_generation); } // A sample of keys must still be found after the rotations. diff --git a/tests/unit/superblock_test.c b/tests/unit/superblock_test.c index 792148cb..9452602b 100644 --- a/tests/unit/superblock_test.c +++ b/tests/unit/superblock_test.c @@ -96,8 +96,7 @@ CTEST2(superblock, test_format_sets_fresh_state) superblock_tree_record rec; superblock_get_tree_record(&ctx, &rec); ASSERT_EQUAL(0, rec.root_addr); // empty tree - ASSERT_EQUAL(SUPERBLOCK_NO_INCORPORATED_GENERATION, - rec.incorporated_generation); + ASSERT_EQUAL(0, rec.first_unincorporated_generation); // replay from the start superblock_context_deinit(&ctx); rc = @@ -127,8 +126,7 @@ CTEST2(superblock, test_snapshot_persists_state) superblock_log_head live = { .addr = 0x6000, .meta_addr = 0x8000, .magic = 0x11}; - superblock_snapshot_tree( - &ctx, 0x4000, SUPERBLOCK_NO_INCORPORATED_GENERATION, live); + superblock_snapshot_tree(&ctx, 0x4000, 0, live); rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); ASSERT_FALSE( @@ -209,7 +207,7 @@ CTEST2(superblock, test_two_log_checkpoint_transitions) ASSERT_TRUE(SUCCESS(rc)); superblock_get_tree_record(&ctx, &got); ASSERT_EQUAL(0x4400, got.root_addr); - ASSERT_EQUAL(9, got.incorporated_generation); + ASSERT_EQUAL(9, got.first_unincorporated_generation); ASSERT_EQUAL(TEST_LOG_L2.meta_addr, got.live_log.meta_addr); ASSERT_TRUE(SUPERBLOCK_NO_LOG(got.sealed_log)); superblock_context_deinit(&ctx); @@ -278,10 +276,7 @@ CTEST2(superblock, test_torn_write_falls_back_to_older_generation) // After format the image is gen 2 in slot 1, so this make_durable targets // slot 0. Snapshot a nonempty root with no live log. - superblock_snapshot_tree(&ctx, - 0x4000, - SUPERBLOCK_NO_INCORPORATED_GENERATION, - (superblock_log_head){0}); + superblock_snapshot_tree(&ctx, 0x4000, 0, (superblock_log_head){0}); rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); superblock_context_deinit(&ctx); From 0904e1ca66b0606b4ae3d7c7bb098361b2bc771a Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Tue, 28 Jul 2026 12:20:12 -0700 Subject: [PATCH 40/64] core cleanup Signed-off-by: Rob Johnson --- src/core.c | 59 +++++++++++++++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/src/core.c b/src/core.c index a911e622..9d11d594 100644 --- a/src/core.c +++ b/src/core.c @@ -162,8 +162,8 @@ core_capture_checkpoint_cut(core_handle *spl, /* * The first generation not folded into the root -- the exclusive replay * bound. When nothing has been retired, memtable_generation_retired() is - * UINT64_MAX and this wraps to 0 ("replay from generation 0"), so no sentinel - * is needed. + * UINT64_MAX and this wraps to 0 ("replay from generation 0"), so no + * sentinel is needed. */ *first_unincorporated_generation = retired_generation + 1; return STATUS_OK; @@ -181,18 +181,19 @@ core_log_to_superblock(log_head info) } /* - * Advance the durable tree to the current COW root and make it durable: capture - * the root, make its pages durable, snapshot it into the superblock (with the - * incorporated-generation cut, the given live log carried forward, and the - * sealed slot cleared -- a snapshot happens only after the sealed log is folded - * in), then release the previously published root. snapshot_tree invalidates - * the persisted allocation state; a clean unmount revalidates it in a later - * step. Used by mkfs (empty root, records the new live log), checkpoint - * completion, and unmount (Part A). The snapshot's owned reference is - * transferred to the durable record on success. + * Commit the trunk's current COW root as the new durable tree root: capture the + * root, make its pages durable, snapshot it into the superblock (recording the + * first unincorporated generation, carrying the given live log forward, and + * clearing the sealed slot -- a snapshot happens only after the sealed log is + * folded in), then release the previously published root. snapshot_tree + * invalidates the persisted allocation state; a clean unmount revalidates it in + * a later step. Used by mkfs (empty root, records the new live log), + * checkpoint completion, and unmount (Part A). The snapshot's owned reference + * is transferred to the durable record on success. */ static platform_status -core_publish_root_record(core_handle *spl, superblock_log_head live_log) +core_checkpoint_commit_current_root(core_handle *spl, + superblock_log_head live_log) { platform_status rc; trunk_snapshot snapshot; @@ -281,7 +282,7 @@ core_publish_root_record(core_handle *spl, superblock_log_head live_log) trunk_snapshot_release(&spl->trunk_context, &old_snapshot); if (!SUCCESS(release_rc)) { platform_error_log( - "core_publish_root_record: trunk_snapshot_release " + "core_commit_current_root: trunk_snapshot_release " "failed for old root addr %lu: %s\n", old_root_addr, platform_status_to_string(release_rc)); @@ -393,7 +394,7 @@ core_checkpoint_maybe_begin(core_handle *spl) * newly enter, the old log -- making the subsequent seal safe. */ static void -core_rotate_log_virtual(void *arg, uint64 finalized_generation) +core_rotate_log(void *arg, uint64 finalized_generation) { core_handle *spl = arg; @@ -463,7 +464,7 @@ core_maybe_complete_checkpoint(core_handle *spl) } platform_status rc = - core_publish_root_record(spl, core_log_to_superblock(live)); + core_checkpoint_commit_current_root(spl, core_log_to_superblock(live)); if (SUCCESS(rc)) { // The sealed log's entries are now durably in the root; free its extents. log_dec_ref(spl->cc, &sealed); @@ -487,14 +488,14 @@ core_maybe_complete_checkpoint(core_handle *spl) } /* - * Resolve any in-flight checkpoint during a quiesced shutdown, before the - * unmount/destroy publish. No locking: the caller has quiesced all inserts and - * incorporations. A completed checkpoint (INCORPORATING/COMPLETING) is - * normally already reaped by the quiesce drain; the residual cases below are - * defensive. + * Release any resources of an in-flight checkpoint during a quiesced shutdown, + * before the unmount/destroy publish. No locking: the caller has quiesced all + * inserts and incorporations. A completed checkpoint + * (INCORPORATING/COMPLETING) is normally already reaped by the quiesce drain; + * the residual cases below are defensive. */ static void -core_finish_checkpoint_for_shutdown(core_handle *spl) +core_checkpoint_cleanup_for_shutdown(core_handle *spl) { core_checkpoint_state *cp = &spl->checkpoint; switch (cp->phase) { @@ -2204,7 +2205,7 @@ core_mkfs(core_handle *spl, } // Swap the checkpoint's live log in from inside the rotation critical // section. - spl->mt_ctxt.rotate = core_rotate_log_virtual; + spl->mt_ctxt.rotate = core_rotate_log; // set up the log if (spl->cfg.use_log) { @@ -2243,9 +2244,9 @@ core_mkfs(core_handle *spl, if (spl->cfg.use_log) { live_log = core_log_to_superblock(log_get_head(spl->log)); } - rc = core_publish_root_record(spl, live_log); + rc = core_checkpoint_commit_current_root(spl, live_log); if (!SUCCESS(rc)) { - platform_error_log("core_mkfs: core_publish_root_record failed: %s\n", + platform_error_log("core_mkfs: core_commit_current_root failed: %s\n", platform_status_to_string(rc)); goto deinit_stats; } @@ -2367,7 +2368,7 @@ core_mount(core_handle *spl, } // Swap the checkpoint's live log in from inside the rotation critical // section. - spl->mt_ctxt.rotate = core_rotate_log_virtual; + spl->mt_ctxt.rotate = core_rotate_log; if (spl->cfg.use_log) { spl->log = shard_log_create( @@ -2628,7 +2629,7 @@ core_checkpoint(core_handle *spl) // --- Complete: advance the durable root to the incorporated state, keeping // the new live log and clearing the sealed slot. --- - rc = core_publish_root_record(spl, new_live); + rc = core_checkpoint_commit_current_root(spl, new_live); if (!SUCCESS(rc)) { return rc; } @@ -2657,7 +2658,7 @@ core_unmount(core_handle *spl) core_quiesce_for_shutdown(spl); // Reclaim any in-flight checkpoint's logs before the unmount publish. - core_finish_checkpoint_for_shutdown(spl); + core_checkpoint_cleanup_for_shutdown(spl); /* * The clean-unmount root incorporates everything, so the live log is fully @@ -2671,7 +2672,7 @@ core_unmount(core_handle *spl) * live or sealed log at rest). Allocation state stays invalid here; it * becomes valid only in Part B, after the map is persisted. */ - rc = core_publish_root_record(spl, (superblock_log_head){0}); + rc = core_checkpoint_commit_current_root(spl, (superblock_log_head){0}); if (!SUCCESS(rc)) { platform_error_log("core_unmount: failed to publish unmount root: %s\n", platform_status_to_string(rc)); @@ -2725,7 +2726,7 @@ core_destroy(core_handle *spl) core_quiesce_for_shutdown(spl); // Reclaim any in-flight checkpoint's logs before teardown. - core_finish_checkpoint_for_shutdown(spl); + core_checkpoint_cleanup_for_shutdown(spl); /* * Release the reference the published tree record holds on its root before From 7f954e70faedc2d9649ccfdbcb1a1f0ec8191eb8 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Tue, 28 Jul 2026 13:25:20 -0700 Subject: [PATCH 41/64] core cleanup Signed-off-by: Rob Johnson --- src/core.c | 30 +++++++++++++----------------- src/memtable.c | 5 ++++- src/memtable.h | 10 +++++++--- tests/functional/btree_test.c | 5 +++-- 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/core.c b/src/core.c index 9d11d594..e1aeaab3 100644 --- a/src/core.c +++ b/src/core.c @@ -141,7 +141,7 @@ core_close_log_stream_if_enabled(core_handle *spl, *----------------------------------------------------------------------------- */ static platform_status -core_capture_checkpoint_cut(core_handle *spl, +core_checkpoint_capture_cut(core_handle *spl, trunk_snapshot *snapshot, uint64 *first_unincorporated_generation) { @@ -174,7 +174,7 @@ core_capture_checkpoint_cut(core_handle *spl, * (layout-identical but module-independent) superblock_log_head. */ static superblock_log_head -core_log_to_superblock(log_head info) +core_log_to_superblock_log_head(log_head info) { return (superblock_log_head){ .addr = info.addr, .meta_addr = info.meta_addr, .magic = info.magic}; @@ -210,7 +210,7 @@ core_checkpoint_commit_current_root(core_handle *spl, return rc; } - rc = core_capture_checkpoint_cut( + rc = core_checkpoint_capture_cut( spl, &snapshot, &first_unincorporated_generation); if (!SUCCESS(rc)) { goto unlock_checkpoint; @@ -463,8 +463,8 @@ core_maybe_complete_checkpoint(core_handle *spl) return STATUS_OK; } - platform_status rc = - core_checkpoint_commit_current_root(spl, core_log_to_superblock(live)); + platform_status rc = core_checkpoint_commit_current_root( + spl, core_log_to_superblock_log_head(live)); if (SUCCESS(rc)) { // The sealed log's entries are now durably in the root; free its extents. log_dec_ref(spl->cc, &sealed); @@ -2196,6 +2196,7 @@ core_mkfs(core_handle *spl, spl->heap_id, cc, mt_cfg, + core_rotate_log, core_memtable_flush_virtual, spl); if (!SUCCESS(rc)) { @@ -2203,9 +2204,6 @@ core_mkfs(core_handle *spl, platform_status_to_string(rc)); goto deinit_superblock; } - // Swap the checkpoint's live log in from inside the rotation critical - // section. - spl->mt_ctxt.rotate = core_rotate_log; // set up the log if (spl->cfg.use_log) { @@ -2242,7 +2240,7 @@ core_mkfs(core_handle *spl, // it durably. No sealed log at mkfs. superblock_log_head live_log = {0}; if (spl->cfg.use_log) { - live_log = core_log_to_superblock(log_get_head(spl->log)); + live_log = core_log_to_superblock_log_head(log_get_head(spl->log)); } rc = core_checkpoint_commit_current_root(spl, live_log); if (!SUCCESS(rc)) { @@ -2357,6 +2355,7 @@ core_mount(core_handle *spl, spl->heap_id, cc, mt_cfg, + core_rotate_log, core_memtable_flush_virtual, spl, resume_generation); @@ -2366,9 +2365,6 @@ core_mount(core_handle *spl, platform_status_to_string(rc)); goto deinit_superblock; } - // Swap the checkpoint's live log in from inside the rotation critical - // section. - spl->mt_ctxt.rotate = core_rotate_log; if (spl->cfg.use_log) { spl->log = shard_log_create( @@ -2416,10 +2412,10 @@ core_mount(core_handle *spl, * silently reverting to this now-stale root. The root is unchanged; a clean * mount has no prior live log, so the sealed slot stays empty. */ - superblock_log_cut(&spl->superblock, - spl->cfg.use_log - ? core_log_to_superblock(log_get_head(spl->log)) - : (superblock_log_head){0}); + superblock_log_cut( + &spl->superblock, + spl->cfg.use_log ? core_log_to_superblock_log_head(log_get_head(spl->log)) + : (superblock_log_head){0}); rc = superblock_make_durable(&spl->superblock); if (!SUCCESS(rc)) { platform_error_log("core_mount: mark-dirty superblock_make_durable " @@ -2602,7 +2598,7 @@ core_checkpoint(core_handle *spl) if (SUCCESS(rc)) { // Cut the log: the just-sealed live log becomes the sealed slot and // the fresh log becomes live. Root unchanged until completion. - new_live = core_log_to_superblock(log_get_head(spl->log)); + new_live = core_log_to_superblock_log_head(log_get_head(spl->log)); superblock_log_cut(&spl->superblock, new_live); rc = superblock_make_durable(&spl->superblock); } diff --git a/src/memtable.c b/src/memtable.c index 7a0d2310..d26f9021 100644 --- a/src/memtable.c +++ b/src/memtable.c @@ -376,6 +376,7 @@ memtable_context_init_at_generation(memtable_context *ctxt, platform_heap_id hid, cache *cc, memtable_config *cfg, + process_fn rotate, process_fn process, void *process_ctxt, uint64 first_generation) @@ -439,6 +440,7 @@ memtable_context_init_at_generation(memtable_context *ctxt, ctxt->is_empty = TRUE; + ctxt->rotate = rotate; ctxt->process = process; ctxt->process_ctxt = process_ctxt; @@ -450,11 +452,12 @@ memtable_context_init(memtable_context *ctxt, platform_heap_id hid, cache *cc, memtable_config *cfg, + process_fn rotate, process_fn process, void *process_ctxt) { return memtable_context_init_at_generation( - ctxt, hid, cc, cfg, process, process_ctxt, 0); + ctxt, hid, cc, cfg, rotate, process, process_ctxt, 0); } void diff --git a/src/memtable.h b/src/memtable.h index 3ab8ef37..29931eaf 100644 --- a/src/memtable.h +++ b/src/memtable.h @@ -124,9 +124,6 @@ typedef struct memtable_context { task_system *ts; platform_heap_id *hid; - process_fn process; - void *process_ctxt; - /* * Optional callback invoked inside the rotation critical section (insert * lock held exclusively), after a memtable is finalized and the generation @@ -135,6 +132,11 @@ typedef struct memtable_context { * Receives process_ctxt and the just-finalized generation. NULL disables. */ process_fn rotate; + /* Process a rotated memtable _outside the critical section of the rotation. + */ + process_fn process; + void *process_ctxt; + // batch distributed read/write locks protect the generation and // generation_retired counters @@ -217,6 +219,7 @@ memtable_context_init(memtable_context *ctxt, platform_heap_id hid, cache *cc, memtable_config *cfg, + process_fn rotate, process_fn process, void *process_ctxt); @@ -234,6 +237,7 @@ memtable_context_init_at_generation(memtable_context *ctxt, platform_heap_id hid, cache *cc, memtable_config *cfg, + process_fn rotate, process_fn process, void *process_ctxt, uint64 first_generation); diff --git a/tests/functional/btree_test.c b/tests/functional/btree_test.c index 18f0efeb..d7a8c8f3 100644 --- a/tests/functional/btree_test.c +++ b/tests/functional/btree_test.c @@ -84,7 +84,7 @@ test_memtable_generation_init(cache *cc, mt_cfg.max_memtables = max_memtables; platform_status rc = memtable_context_init( - &mt_ctxt, hid, cc, &mt_cfg, test_btree_process_noop, NULL); + &mt_ctxt, hid, cc, &mt_cfg, NULL, test_btree_process_noop, NULL); if (!SUCCESS(rc)) { return rc; } @@ -119,6 +119,7 @@ test_memtable_generation_init(cache *cc, hid, cc, &mt_cfg, + NULL, test_btree_process_noop, NULL, first_generation); @@ -171,7 +172,7 @@ test_memtable_context_create(cache *cc, ctxt->cfg = cfg; ctxt->heap_id = hid; platform_status rc = memtable_context_init( - &ctxt->mt_ctxt, hid, cc, cfg->mt_cfg, test_btree_process_noop, NULL); + &ctxt->mt_ctxt, hid, cc, cfg->mt_cfg, NULL, test_btree_process_noop, NULL); if (!SUCCESS(rc)) { platform_free(hid, ctxt); return NULL; From cee1ac70b8a923ae7b1be00a74ee74c9621a7525 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Tue, 28 Jul 2026 13:28:26 -0700 Subject: [PATCH 42/64] core cleanup Signed-off-by: Rob Johnson --- src/core.c | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/src/core.c b/src/core.c index e1aeaab3..efa7c019 100644 --- a/src/core.c +++ b/src/core.c @@ -2517,18 +2517,6 @@ core_quiesce_for_shutdown(core_handle *spl) core_report_unincorporated_memtables(spl); } -static void -core_teardown_after_shutdown(core_handle *spl) -{ - // Keep this after checkpoint publication: it supplies the generation cut. - memtable_context_deinit(&spl->mt_ctxt); - - // flush all dirty pages in the cache. The live log has already been sealed - // by the caller (core_seal_live_log); its extents are freed after this - // flush. - cache_flush(spl->cc); -} - /* * Seal the live log at shutdown -- finalizes its pages and frees the handle -- * and return its identity so the caller can free the extents with log_dec_ref() @@ -2674,9 +2662,13 @@ core_unmount(core_handle *spl) platform_status_to_string(rc)); } - core_teardown_after_shutdown(spl); - // Free the sealed live log's extents now that the cache is flushed, before - // the map is persisted (Part B) so the persisted map reflects the free. + // Keep this after publication above: it supplies the generation cut. + memtable_context_deinit(&spl->mt_ctxt); + + // Flush all dirty pages. The live log has already been sealed (above); free + // its extents now that the cache is flushed, before the map is persisted + // (Part B) so the persisted map reflects the free. + cache_flush(spl->cc); log_dec_ref(spl->cc, &live_log); /* * Release the context's live root reference before persisting the map, so @@ -2747,7 +2739,8 @@ core_destroy(core_handle *spl) // Discard the live log too: seal (frees the handle), then free its extents // after the cache flush. log_head live_log = core_seal_live_log(spl); - core_teardown_after_shutdown(spl); + memtable_context_deinit(&spl->mt_ctxt); + cache_flush(spl->cc); log_dec_ref(spl->cc, &live_log); trunk_context_deinit(&spl->trunk_context); From 9c9765e95bd2bee282a7e05115b1d2cdbaf72544 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Tue, 28 Jul 2026 16:59:01 -0700 Subject: [PATCH 43/64] make memtable_force_rotation act like a normal rotation Signed-off-by: Rob Johnson --- src/core.c | 13 ++++++------- src/memtable.c | 19 +++++++++++++++++-- src/memtable.h | 13 ++++++++++++- tests/unit/splinterdb_optimize_test.c | 5 ++--- tests/unit/splinterdb_quick_test.c | 5 ++--- 5 files changed, 39 insertions(+), 16 deletions(-) diff --git a/src/core.c b/src/core.c index efa7c019..a5d17775 100644 --- a/src/core.c +++ b/src/core.c @@ -2502,12 +2502,12 @@ core_quiesce_for_shutdown(core_handle *spl) if (!memtable_is_empty(&spl->mt_ctxt)) { /* - * memtable_force_finalize is not thread safe. Note also, we do not hold - * the insert lock or rotate while flushing the memtable. + * memtable_force_rotation is not thread safe. It dispatches the flush + * itself (via the process callback), which also resolves any checkpoint + * log cut its rotate hook just made. That callback may arm a fresh + * checkpoint; core_checkpoint_cleanup_for_shutdown() discards it. */ - - uint64 generation = memtable_force_finalize(&spl->mt_ctxt); - core_memtable_flush(spl, generation); + memtable_force_rotation(&spl->mt_ctxt); } // finish any outstanding tasks and destroy task system for this table. @@ -2603,8 +2603,7 @@ core_checkpoint(core_handle *spl) // --- Incorporate: fold everything logged so far into the trunk, advancing // the in-memory COW root. Synchronous (no concurrent inserts). --- if (!memtable_is_empty(&spl->mt_ctxt)) { - uint64 generation = memtable_force_finalize(&spl->mt_ctxt); - core_memtable_flush(spl, generation); + memtable_force_rotation(&spl->mt_ctxt); // dispatches the flush itself } rc = task_perform_until_quiescent(spl->ts); if (!SUCCESS(rc)) { diff --git a/src/memtable.c b/src/memtable.c index d26f9021..13793514 100644 --- a/src/memtable.c +++ b/src/memtable.c @@ -83,7 +83,7 @@ memtable_end_insert(memtable_context *ctxt) /* * Exclude all inserts, including one that has already acquired its shared * insert lock, so the caller can safely mutate generation state. Used by - * memtable_force_finalize(); the normal rotation path instead upgrades its own + * memtable_force_rotation(); the normal rotation path instead upgrades its own * insert lock via memtable_try_begin_insert_rotation(). Pair with * memtable_unblock_inserts(). */ @@ -310,7 +310,7 @@ memtable_mark_incorporation_failed(memtable *mt, platform_status status) } uint64 -memtable_force_finalize(memtable_context *ctxt) +memtable_force_rotation(memtable_context *ctxt) { memtable_block_inserts(ctxt); @@ -323,7 +323,22 @@ memtable_force_finalize(memtable_context *ctxt) <= ctxt->cfg.max_memtables); memtable_mark_empty(ctxt); + /* + * Inserts are still excluded, exactly as in the natural (fullness-triggered) + * rotation: all in-flight inserts (and their log writes) have drained and + * none can start. See memtable_maybe_rotate_and_begin_insert() for why this + * is the one safe point to swap the checkpoint's live log. + */ + if (ctxt->rotate != NULL) { + ctxt->rotate(ctxt->process_ctxt, current_generation); + } + memtable_unblock_inserts(ctxt); + + // Dispatch the rotated memtable, outside the critical section, exactly as + // the natural rotation path does. + memtable_process(ctxt, current_generation); + return current_generation; } diff --git a/src/memtable.h b/src/memtable.h index 29931eaf..443cd5cf 100644 --- a/src/memtable.h +++ b/src/memtable.h @@ -205,8 +205,19 @@ memtable_mark_incorporation_failed(memtable *mt, platform_status status); const char * memtable_state_string(memtable_state state); +/* + * Rotate the current memtable now, regardless of fullness. Performs the same + * sequence as the natural (fullness-triggered) rotation: finalize the memtable, + * advance the generation, invoke the rotate callback under insert exclusion, + * then -- once inserts are unblocked -- invoke the process callback to dispatch + * the rotated memtable. Returns the finalized generation. + * + * Callers therefore need do nothing further: whatever the rotate callback + * started (e.g. core's checkpoint log cut) is resolved by the process callback, + * just as it is for a natural rotation. + */ uint64 -memtable_force_finalize(memtable_context *ctxt); +memtable_force_rotation(memtable_context *ctxt); void memtable_init(memtable *mt, cache *cc, memtable_config *cfg, uint64 generation); diff --git a/tests/unit/splinterdb_optimize_test.c b/tests/unit/splinterdb_optimize_test.c index a8587cf6..215e8219 100644 --- a/tests/unit/splinterdb_optimize_test.c +++ b/tests/unit/splinterdb_optimize_test.c @@ -284,9 +284,8 @@ create_optimize_cfg(splinterdb_config *out_cfg, data_config *default_data_cfg) static void force_flush_current_memtable(splinterdb *kvsb) { - core_handle *core = (core_handle *)splinterdb_get_trunk_handle(kvsb); - uint64 generation = memtable_force_finalize(&core->mt_ctxt); - core->mt_ctxt.process(core->mt_ctxt.process_ctxt, generation); + core_handle *core = (core_handle *)splinterdb_get_trunk_handle(kvsb); + memtable_force_rotation(&core->mt_ctxt); platform_status rc = task_perform_until_quiescent(core->ts); ASSERT_TRUE(SUCCESS(rc)); } diff --git a/tests/unit/splinterdb_quick_test.c b/tests/unit/splinterdb_quick_test.c index 512b902b..09fa6ddc 100644 --- a/tests/unit/splinterdb_quick_test.c +++ b/tests/unit/splinterdb_quick_test.c @@ -1810,9 +1810,8 @@ static uint64 force_flush_current_memtable(splinterdb *kvsb) { core_handle *core = (core_handle *)splinterdb_get_trunk_handle(kvsb); - uint64 generation = memtable_force_finalize(&core->mt_ctxt); - core->mt_ctxt.process(core->mt_ctxt.process_ctxt, generation); - platform_status rc = task_perform_until_quiescent(core->ts); + uint64 generation = memtable_force_rotation(&core->mt_ctxt); + platform_status rc = task_perform_until_quiescent(core->ts); ASSERT_TRUE(SUCCESS(rc)); return generation; } From 416a1abd110175b82a34cf776c91fa267bb8d52c Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Wed, 29 Jul 2026 10:31:51 -0700 Subject: [PATCH 44/64] simplify core_checkpoint and fix a bug Signed-off-by: Rob Johnson --- src/core.c | 325 +++++++++++++++++++++-------------- src/core.h | 39 +++-- src/superblock.c | 30 ++-- src/superblock.h | 54 ++++-- tests/unit/splinter_test.c | 14 +- tests/unit/superblock_test.c | 72 ++++++-- 6 files changed, 354 insertions(+), 180 deletions(-) diff --git a/src/core.c b/src/core.c index a5d17775..c44d1534 100644 --- a/src/core.c +++ b/src/core.c @@ -170,14 +170,36 @@ core_checkpoint_capture_cut(core_handle *spl, } /* - * Translate between the log module's log_head and the superblock's - * (layout-identical but module-independent) superblock_log_head. + * Translate the log module's log_head into the superblock's descriptor, adding + * the coverage information the log module does not track: start_generation is + * the first memtable generation whose entries went to this log. */ static superblock_log_head -core_log_to_superblock_log_head(log_head info) +core_log_to_superblock_log_head(log_head info, uint64 start_generation) { - return (superblock_log_head){ - .addr = info.addr, .meta_addr = info.meta_addr, .magic = info.magic}; + return (superblock_log_head){.addr = info.addr, + .meta_addr = info.meta_addr, + .magic = info.magic, + .start_generation = start_generation}; +} + +/* + * The superblock descriptor for the log currently receiving inserts. Read + * under checkpoint_state_lock, which core_rotate_log() holds while it swaps the + * live log, so this can never observe a half-swapped state (recording the + * just-sealed log as live would make recovery miss every post-cut insert). + */ +static superblock_log_head +core_current_live_log(core_handle *spl) +{ + if (!spl->cfg.use_log) { + return (superblock_log_head){0}; + } + platform_mutex_lock(&spl->checkpoint_state_lock); + superblock_log_head live = core_log_to_superblock_log_head( + log_get_head(spl->log), spl->live_log_start_generation); + platform_mutex_unlock(&spl->checkpoint_state_lock); + return live; } /* @@ -400,37 +422,84 @@ core_rotate_log(void *arg, uint64 finalized_generation) platform_mutex_lock(&spl->checkpoint_state_lock); if (spl->checkpoint.phase == CORE_CHECKPOINT_PENDING) { - spl->checkpoint.log_to_seal = spl->log; - spl->checkpoint.sealed_head = log_get_head(spl->log); - spl->log = spl->checkpoint.pending_log; - spl->checkpoint.pending_log = NULL; - spl->checkpoint.cut_generation = finalized_generation; - spl->checkpoint.phase = CORE_CHECKPOINT_SEALING; + spl->checkpoint.log_to_seal = spl->log; + spl->checkpoint.sealed_head = log_get_head(spl->log); + spl->log = spl->checkpoint.pending_log; + spl->checkpoint.pending_log = NULL; + /* + * Hand off generation coverage: the sealed log received everything up to + * and including finalized_generation, so the new live log starts at the + * next one. + */ + spl->checkpoint.sealed_start_generation = spl->live_log_start_generation; + spl->live_log_start_generation = finalized_generation + 1; + spl->checkpoint.cut_generation = finalized_generation; + spl->checkpoint.phase = CORE_CHECKPOINT_SEALING; } platform_mutex_unlock(&spl->checkpoint_state_lock); } /* * Begin, step 3 (just after the rotation critical section): seal the - * swapped-out log. The swap drained and excluded all writers, so sealing is - * safe. Seal finalizes the pages (dirty) and frees the handle; durability is - * deferred to completion (or a future sync), so no barrier here. + * swapped-out log and publish the cut. The swap drained and excluded all + * writers, so sealing is safe. + * + * Publishing here is what makes the cut crash-safe. The rotation moved inserts + * to the new live log, but the superblock still names the old one, so until this + * runs a crash would lose everything written to the new log. Order matters: the + * sealed log's pages must be durable before the superblock names it as sealed, + * since recovery replays it as-is. */ static void core_checkpoint_seal_cut(core_handle *spl) { platform_mutex_lock(&spl->checkpoint_state_lock); - log_handle *to_seal = NULL; + log_handle *to_seal = NULL; + superblock_log_head sealed = {0}; + superblock_log_head live = {0}; if (spl->checkpoint.phase == CORE_CHECKPOINT_SEALING) { to_seal = spl->checkpoint.log_to_seal; spl->checkpoint.log_to_seal = NULL; spl->checkpoint.phase = CORE_CHECKPOINT_INCORPORATING; + sealed = core_log_to_superblock_log_head( + spl->checkpoint.sealed_head, spl->checkpoint.sealed_start_generation); + live = core_log_to_superblock_log_head(spl->checkpoint.live_head, + spl->live_log_start_generation); } platform_mutex_unlock(&spl->checkpoint_state_lock); - if (to_seal != NULL) { - log_seal(to_seal); + if (to_seal == NULL) { + return; + } + log_seal(to_seal); + + /* + * Serialize against any other superblock publisher (the superblock context + * is not thread safe). On failure the checkpoint still proceeds: the + * completion publish will record the correct final state; only this + * crash-protection window is left uncovered. + */ + platform_status rc = platform_mutex_lock(&spl->checkpoint_lock); + if (!SUCCESS(rc)) { + platform_error_log("core_checkpoint_seal_cut: lock failed: %s\n", + platform_status_to_string(rc)); + return; + } + rc = cache_writeback_dirty(spl->cc); + if (SUCCESS(rc)) { + rc = cache_durable_barrier(spl->cc); } + if (SUCCESS(rc)) { + superblock_log_cut(&spl->superblock, sealed, live); + rc = superblock_make_durable(&spl->superblock); + } + if (!SUCCESS(rc)) { + platform_error_log("core_checkpoint_seal_cut: failed to publish the log " + "cut: %s\n", + platform_status_to_string(rc)); + } + platform_status unlock_rc = platform_mutex_unlock(&spl->checkpoint_lock); + platform_assert_status_ok(unlock_rc); } /* @@ -450,11 +519,12 @@ core_maybe_complete_checkpoint(core_handle *spl) uint64 first_unincorporated = memtable_generation_retired(&spl->mt_ctxt) + 1; bool32 complete = spl->checkpoint.phase == CORE_CHECKPOINT_INCORPORATING && first_unincorporated > spl->checkpoint.cut_generation; - log_head sealed = {0}; - log_head live = {0}; + log_head sealed = {0}; + superblock_log_head live = {0}; if (complete) { - sealed = spl->checkpoint.sealed_head; - live = spl->checkpoint.live_head; + sealed = spl->checkpoint.sealed_head; + live = core_log_to_superblock_log_head(spl->checkpoint.live_head, + spl->live_log_start_generation); spl->checkpoint.phase = CORE_CHECKPOINT_COMPLETING; } platform_mutex_unlock(&spl->checkpoint_state_lock); @@ -463,8 +533,7 @@ core_maybe_complete_checkpoint(core_handle *spl) return STATUS_OK; } - platform_status rc = core_checkpoint_commit_current_root( - spl, core_log_to_superblock_log_head(live)); + platform_status rc = core_checkpoint_commit_current_root(spl, live); if (SUCCESS(rc)) { // The sealed log's entries are now durably in the root; free its extents. log_dec_ref(spl->cc, &sealed); @@ -786,10 +855,16 @@ core_memtable_compact(core_handle *spl, uint64 generation, const threadid tid) } core_memtable_iterator_deinit(&btree_itor); + /* + * A forced rotation (see memtable_force_rotation(), used by core_checkpoint() + * when nothing rotates on its own) can finalize a memtable that received no + * inserts. btree_pack() already defines this case: an empty input yields + * num_tuples == 0 and root_addr == 0, allocating no page. The generation + * still retires; core_memtable_incorporate() recognizes the missing branch + * and skips the trunk incorporation. + */ new_branch->root_addr = req.root_addr; - platform_assert(req.num_tuples > 0); - btree_pack_req_deinit(&req, spl->heap_id); if (spl->cfg.use_stats) { uint64 comp_time = platform_timestamp_elapsed(comp_start); @@ -879,19 +954,30 @@ core_memtable_incorporate(core_handle *spl, if (spl->cfg.use_stats) { flush_start = platform_get_timestamp(); } - rc = trunk_incorporate_prepare(&spl->trunk_context, cmt->branch.root_addr); - if (!SUCCESS(rc)) { - platform_error_log("trunk_incorporate_prepare failed: %s\n", - platform_status_to_string(rc)); - core_close_log_stream_if_enabled(spl, &stream); - core_memtable_mark_incorporation_failed(spl, generation, rc); - return rc; - } - btree_dec_ref( - spl->cc, spl->cfg.btree_cfg, cmt->branch.root_addr, PAGE_TYPE_BRANCH); - if (spl->cfg.use_stats) { - spl->stats[tid].memtable_flush_wait_time_ns += - platform_timestamp_elapsed(cmt->wait_start); + /* + * A forced rotation can retire a generation that received no inserts, in + * which case core_memtable_compact() produced no branch (root_addr == 0). + * There is nothing to fold into the trunk, and the trunk_incorporate_*() + * calls require a real branch (trunk_incorporate_prepare() asserts + * branch_addr != 0), so skip them; the generation still retires below. + */ + bool32 has_branch = (cmt->branch.root_addr != 0); + if (has_branch) { + rc = trunk_incorporate_prepare(&spl->trunk_context, + cmt->branch.root_addr); + if (!SUCCESS(rc)) { + platform_error_log("trunk_incorporate_prepare failed: %s\n", + platform_status_to_string(rc)); + core_close_log_stream_if_enabled(spl, &stream); + core_memtable_mark_incorporation_failed(spl, generation, rc); + return rc; + } + btree_dec_ref( + spl->cc, spl->cfg.btree_cfg, cmt->branch.root_addr, PAGE_TYPE_BRANCH); + if (spl->cfg.use_stats) { + spl->stats[tid].memtable_flush_wait_time_ns += + platform_timestamp_elapsed(cmt->wait_start); + } } core_log_stream_if_enabled( @@ -915,10 +1001,14 @@ core_memtable_incorporate(core_handle *spl, memtable_transition( mt, MEMTABLE_STATE_INCORPORATING, MEMTABLE_STATE_INCORPORATED); memtable_increment_to_generation_retired(&spl->mt_ctxt, generation); - trunk_incorporate_commit(&spl->trunk_context); + if (has_branch) { + trunk_incorporate_commit(&spl->trunk_context); + } memtable_unblock_lookups(&spl->mt_ctxt); - trunk_incorporate_cleanup(&spl->trunk_context); + if (has_branch) { + trunk_incorporate_cleanup(&spl->trunk_context); + } core_close_log_stream_if_enabled(spl, &stream); @@ -1063,6 +1153,12 @@ core_memtable_lookup(core_handle *spl, bool32 memtable_is_compacted; uint64 root_addr = core_memtable_root_addr_for_lookup( spl, generation, &memtable_is_compacted, NULL); + if (memtable_is_compacted && root_addr == 0) { + // A forced rotation can retire an empty generation, whose compacted + // branch has no root page. It holds no tuples, so there is nothing to + // search -- equivalent to finding nothing here. + return STATUS_OK; + } page_type type = memtable_is_compacted ? PAGE_TYPE_BRANCH : PAGE_TYPE_MEMTABLE; @@ -1379,13 +1475,21 @@ core_range_iterator_init(core_handle *spl, bool32 active; uint64 root_addr = core_memtable_root_addr_for_lookup(spl, mt_gen, &compacted, &active); - range_itor->compacted[range_itor->num_branches] = compacted; // Only READY memtables can be modified while this iterator is live. + // Determined before the empty-generation skip below so that skipping + // cannot affect which generation is treated as the first one. if (range_itor->num_branches == 0) { first_memtable_copy_nodes = active; } else { debug_assert(!active); } + if (compacted && root_addr == 0) { + // A forced rotation can retire an empty generation, whose compacted + // branch has no root page. It contributes no tuples, so there is + // nothing to merge from it. + continue; + } + range_itor->compacted[range_itor->num_branches] = compacted; if (compacted) { btree_inc_ref(spl->cc, spl->cfg.btree_cfg, root_addr); } else { @@ -2237,12 +2341,9 @@ core_mkfs(core_handle *spl, } // Establish the initial (empty) tree record, recording the live log; publish - // it durably. No sealed log at mkfs. - superblock_log_head live_log = {0}; - if (spl->cfg.use_log) { - live_log = core_log_to_superblock_log_head(log_get_head(spl->log)); - } - rc = core_checkpoint_commit_current_root(spl, live_log); + // it durably. No sealed log at mkfs; the log starts at generation 0. + spl->live_log_start_generation = 0; + rc = core_checkpoint_commit_current_root(spl, core_current_live_log(spl)); if (!SUCCESS(rc)) { platform_error_log("core_mkfs: core_commit_current_root failed: %s\n", platform_status_to_string(rc)); @@ -2410,12 +2511,13 @@ core_mount(core_handle *spl, * allocation state, before any allocation diverges the in-memory map from * disk. A crash after this forces the next mount into recovery instead of * silently reverting to this now-stale root. The root is unchanged; a clean - * mount has no prior live log, so the sealed slot stays empty. + * mount has no prior live log, so the sealed slot stays empty. This + * session's log receives generations from the resume generation onward. */ - superblock_log_cut( - &spl->superblock, - spl->cfg.use_log ? core_log_to_superblock_log_head(log_get_head(spl->log)) - : (superblock_log_head){0}); + spl->live_log_start_generation = resume_generation; + superblock_log_cut(&spl->superblock, + (superblock_log_head){0}, + core_current_live_log(spl)); rc = superblock_make_durable(&spl->superblock); if (!SUCCESS(rc)) { platform_error_log("core_mount: mark-dirty superblock_make_durable " @@ -2537,91 +2639,55 @@ core_seal_live_log(core_handle *spl) } /* - * Take a checkpoint: advance the durable root, rotating the log via the two-log - * protocol. Call at a quiescent point (no concurrent inserts). - * - * Two superblock publishes bracket the incorporation, so a crash mid-checkpoint - * recovers to a well-defined state: + * Take a checkpoint: make every modification that completed before this call + * durable in the trunk root, and block until that is done. Safe to call on a + * running system with concurrent inserts. * - * begin -> {root = C_prev, sealed = old live log, live = new log} - * (fold the sealed log's entries into the trunk, advancing the COW root) - * complete -> {root = C_new (incorporates the sealed log), sealed = empty} + * The whole guarantee reduces to a generation bound. Inserts land in the + * currently active memtable generation and generations only advance, so + * everything already inserted is in a generation at or below the one active at + * entry -- call it the target. Once the target has been incorporated, every one + * of those inserts is in the trunk's COW root, and committing that root makes + * them durable. Log rotation is deliberately not part of this: the two-log + * protocol bounds replay and reclaims log extents, which is the automatic + * checkpoint policy's job, not a durability requirement. A checkpoint already + * in flight is therefore irrelevant here and is neither waited on nor disturbed. * - * then the sealed log's extents are freed. With logging disabled it is simply - * an incorporate + durable-root advance (no log to rotate). + * The target is the *active* generation, so it cannot be incorporated until + * something finalizes it. Insert traffic normally does; if none arrives within + * rotation_timeout_ns we force the rotation. Forcing is safe even with an empty + * memtable: that generation retires with no branch at all (see + * core_memtable_compact()). */ platform_status -core_checkpoint(core_handle *spl) +core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns) { - platform_status rc; - superblock_log_head new_live = {0}; - log_head sealed = {0}; - - if (spl->cfg.use_log) { - // --- Begin: seal the live log, start a fresh one, and publish - // {sealed = old live, live = new} without advancing the root. --- - rc = platform_mutex_lock(&spl->checkpoint_lock); - if (!SUCCESS(rc)) { - return rc; - } - - sealed = log_get_head(spl->log); - log_seal(spl->log); // finalizes the pages (dirty), frees the handle - spl->log = NULL; - - // The sealed log must be fully durable before it is recorded as the - // sealed segment (recovery replays it as-is). Flush the finalized pages. - rc = cache_writeback_dirty(spl->cc); - if (SUCCESS(rc)) { - rc = cache_durable_barrier(spl->cc); - } - if (SUCCESS(rc)) { - spl->log = shard_log_create( - spl->cc, (shard_log_config *)spl->cfg.log_cfg, spl->heap_id); - if (spl->log == NULL) { - platform_error_log("core_checkpoint: shard_log_create failed\n"); - rc = STATUS_NO_MEMORY; - } - } - if (SUCCESS(rc)) { - // Cut the log: the just-sealed live log becomes the sealed slot and - // the fresh log becomes live. Root unchanged until completion. - new_live = core_log_to_superblock_log_head(log_get_head(spl->log)); - superblock_log_cut(&spl->superblock, new_live); - rc = superblock_make_durable(&spl->superblock); - } + uint64 target = memtable_generation(&spl->mt_ctxt); - platform_status unlock_rc = platform_mutex_unlock(&spl->checkpoint_lock); - if (SUCCESS(rc) && !SUCCESS(unlock_rc)) { - rc = unlock_rc; - } - if (!SUCCESS(rc)) { - return rc; + uint64 wait = 100; + timestamp deadline = platform_get_timestamp(); + while (memtable_generation_retired(&spl->mt_ctxt) + 1 <= target) { + /* + * Force a rotation only while the target is still the active generation. + * If it has already advanced, the target is finalized and merely needs its + * flush to drain -- forcing again would rotate an unrelated generation. + */ + if (memtable_generation(&spl->mt_ctxt) == target + && rotation_timeout_ns <= platform_timestamp_elapsed(deadline)) + { + memtable_force_rotation(&spl->mt_ctxt); // dispatches the flush itself } + task_perform_one_if_needed(spl->ts, 0); + platform_sleep_ns(wait); + wait = wait > 2048 ? wait : 2 * wait; } - // --- Incorporate: fold everything logged so far into the trunk, advancing - // the in-memory COW root. Synchronous (no concurrent inserts). --- - if (!memtable_is_empty(&spl->mt_ctxt)) { - memtable_force_rotation(&spl->mt_ctxt); // dispatches the flush itself - } - rc = task_perform_until_quiescent(spl->ts); - if (!SUCCESS(rc)) { - return rc; - } - - // --- Complete: advance the durable root to the incorporated state, keeping - // the new live log and clearing the sealed slot. --- - rc = core_checkpoint_commit_current_root(spl, new_live); - if (!SUCCESS(rc)) { - return rc; - } - - // The sealed log's entries are now durably in the root; free its extents. - if (spl->cfg.use_log) { - log_dec_ref(spl->cc, &sealed); - } - return STATUS_OK; + /* + * The target is incorporated. Commit the current root, recording the log + * that is actually live; superblock_snapshot_tree() keeps a sealed log that + * this root does not yet cover. + */ + return core_checkpoint_commit_current_root(spl, core_current_live_log(spl)); } /* @@ -3016,6 +3082,9 @@ core_print_lookup(core_handle *spl, key target, platform_log_handle *log_handle) bool32 memtable_is_compacted; uint64 root_addr = core_memtable_root_addr_for_lookup( spl, mt_gen, &memtable_is_compacted, NULL); + if (memtable_is_compacted && root_addr == 0) { + continue; // empty generation: no branch to look up in + } platform_status rc; rc = btree_lookup(spl->cc, diff --git a/src/core.h b/src/core.h index bc0f065d..c2b1428e 100644 --- a/src/core.h +++ b/src/core.h @@ -129,11 +129,14 @@ typedef enum core_checkpoint_phase { typedef struct core_checkpoint_state { core_checkpoint_phase phase; - log_handle *pending_log; // next live log, pre-created (PENDING) - log_handle *log_to_seal; // old live log awaiting seal (SEALING) - log_head sealed_head; // identity of the sealed log (reclaim) - log_head live_head; // identity of the new live log - uint64 cut_generation; // complete once retired >= this + log_handle *pending_log; // next live log, pre-created (PENDING) + log_handle *log_to_seal; // old live log awaiting seal (SEALING) + log_head sealed_head; // identity of the sealed log (reclaim) + log_head live_head; // identity of the new live log + // First generation the sealed log received; with live_log_start_generation + // this gives the sealed log's coverage for the superblock. + uint64 sealed_start_generation; + uint64 cut_generation; // complete once retired >= this } core_checkpoint_state; typedef struct core_memtable_args { @@ -176,13 +179,20 @@ struct core_handle { /* * Incorporation-driven checkpoint state. checkpoint_state_lock guards the - * fields of `checkpoint` and last_checkpoint_generation; it is only ever - * held for brief, I/O-free updates (never across a barrier), so taking it - * inside the memtable rotation critical section cannot stall inserts on I/O. + * fields of `checkpoint`, last_checkpoint_generation, and + * live_log_start_generation (and is held while `log` is swapped); it is only + * ever held for brief, I/O-free updates (never across a barrier), so taking + * it inside the memtable rotation critical section cannot stall inserts on + * I/O. */ platform_mutex checkpoint_state_lock; core_checkpoint_state checkpoint; uint64 last_checkpoint_generation; + /* + * First memtable generation whose entries went to the current `log`. Recorded + * in the superblock so it can tell which generations each log covers. + */ + uint64 live_log_start_generation; core_stats *stats; @@ -304,11 +314,18 @@ core_mount(core_handle *spl, platform_heap_id hid); /* - * Take a checkpoint: advance the durable root and rotate the log (two-log - * protocol). Must be called at a quiescent point (no concurrent inserts). + * Take a checkpoint: make every modification made before this call durable in + * the trunk root, rotating the log via the two-log protocol. Blocks until the + * checkpoint completes. Safe to call on a running system with concurrent + * inserts; if a checkpoint is already in flight, waits for it and then takes + * another (so the result always covers this caller's modifications). + * + * The log cut requires a memtable rotation, which insert traffic normally + * triggers. If none occurs within rotation_timeout_ns, the rotation is forced. + * Pass 0 to force it immediately. See core.c for the full contract. */ platform_status -core_checkpoint(core_handle *spl); +core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns); platform_status core_unmount(core_handle *spl); diff --git a/src/superblock.c b/src/superblock.c index 6792ad66..06421520 100644 --- a/src/superblock.c +++ b/src/superblock.c @@ -218,12 +218,14 @@ superblock_get_tree_record(const superblock_context *ctx, } void -superblock_log_cut(superblock_context *ctx, superblock_log_head new_live) +superblock_log_cut(superblock_context *ctx, + superblock_log_head sealed, + superblock_log_head new_live) { - // The current live log becomes the sealed log (its entries are being folded - // into the next root); new_live receives subsequent inserts. The persisted - // allocation map no longer matches the (log) state, so invalidate it. - ctx->image->tree.sealed_log = ctx->image->tree.live_log; + // The sealed log's entries are still being folded into the next root; + // new_live receives subsequent inserts. The persisted allocation map no + // longer matches the (log) state, so invalidate it. + ctx->image->tree.sealed_log = sealed; ctx->image->tree.live_log = new_live; ctx->image->allocation_state_addr = 0; } @@ -234,16 +236,24 @@ superblock_snapshot_tree(superblock_context *ctx, uint64 first_unincorporated_generation, superblock_log_head new_live) { - // The durable tree now includes everything folded into root_addr, so the - // sealed log (if any) is done with; new_live is the log carried forward - // (empty at a clean shutdown). Advancing the root diverges the persisted - // allocation map, so invalidate it. + // Advancing the root diverges the persisted allocation map, so invalidate it. ctx->image->tree.root_addr = root_addr; ctx->image->tree.first_unincorporated_generation = first_unincorporated_generation; - ctx->image->tree.sealed_log = (superblock_log_head){0}; ctx->image->tree.live_log = new_live; ctx->image->allocation_state_addr = 0; + + /* + * Drop the sealed log only once the new root covers everything it holds. It + * covers generations up to new_live.start_generation - 1, so it is fully + * incorporated exactly when first_unincorporated_generation reaches + * new_live.start_generation. Otherwise keep it: recovery still needs those + * entries. (A cleared live log means a clean shutdown, where everything is + * incorporated and start_generation is 0, so the sealed slot clears too.) + */ + if (first_unincorporated_generation >= new_live.start_generation) { + ctx->image->tree.sealed_log = (superblock_log_head){0}; + } } void diff --git a/src/superblock.h b/src/superblock.h index 16a2540f..1c4e5a79 100644 --- a/src/superblock.h +++ b/src/superblock.h @@ -38,21 +38,32 @@ #include "platform_io.h" #include "util.h" -#define SUPERBLOCK_FORMAT_MAGIC (0x5344425355504552ULL) // SDBSUPER -#define SUPERBLOCK_FORMAT_VERSION (1) +#define SUPERBLOCK_FORMAT_MAGIC (0x5344425355504552ULL) // SDBSUPER +/* v2 added superblock_log_head.start_generation. */ +#define SUPERBLOCK_FORMAT_VERSION (2) /* The two physical superblock copies live at pages 0 and 1. */ #define SUPERBLOCK_NUM_SLOTS (2) /* - * A log's on-disk head. Mirrors log_head's layout; the superblock stores it - * opaquely and does not depend on the log module. meta_addr == 0 means "no log - * present". + * A log's on-disk head, plus the range of memtable generations it covers. The + * addr/meta_addr/magic triple mirrors log_head's layout; the superblock stores + * it opaquely and does not depend on the log module. meta_addr == 0 means "no + * log present". */ typedef struct ONDISK superblock_log_head { uint64 addr; uint64 meta_addr; uint64 magic; + /* + * First memtable generation whose entries this log received. A log's + * coverage ends where the next log's begins, so the sealed log covers + * [sealed_log.start_generation, live_log.start_generation - 1]. That lets + * the superblock decide on its own when a sealed log has been fully + * incorporated and can be dropped (see superblock_snapshot_tree()), and it + * tells recovery which log to begin replaying from. + */ + uint64 start_generation; } superblock_log_head; /* An empty (absent) log slot: meta_addr == 0. */ @@ -178,21 +189,32 @@ superblock_format(superblock_context *ctx, const allocator_config *cfg); */ /* - * Rotate the log: the current live log becomes the sealed log (its entries are - * being folded into the next root) and new_live receives subsequent inserts. - * Invalidates the allocation state. Used at a checkpoint's begin, and to - * install a fresh live log at mkfs/mount (where there is no prior live log, so - * the sealed slot stays empty). + * Record the log configuration produced by a cut: `sealed` is the just-retired + * log whose entries are still being folded into the next root (empty if there + * is none) and `new_live` receives subsequent inserts. Both slots are given + * explicitly rather than inferred from the current image, so this is correct + * regardless of what else has published in the meantime. Invalidates the + * allocation state. Used at a checkpoint's begin, and to install a fresh live + * log at mkfs/mount (with an empty sealed slot). */ void -superblock_log_cut(superblock_context *ctx, superblock_log_head new_live); +superblock_log_cut(superblock_context *ctx, + superblock_log_head sealed, + superblock_log_head new_live); /* - * Advance the durable tree to root_addr (with first_unincorporated_generation - * the first generation not folded in) and clear the sealed log -- a snapshot is - * taken only after the sealed log has been folded into the root. new_live is - * the log carried forward (empty at a clean shutdown). Invalidates the - * allocation state. Used at a checkpoint's completion and at a clean unmount. + * Advance the durable tree to root_addr, where first_unincorporated_generation + * is the first generation not folded into it, carrying new_live forward as the + * live log (empty at a clean shutdown). Invalidates the allocation state. + * Used at a checkpoint's completion, at a durability checkpoint, and at a clean + * unmount. + * + * The sealed log is dropped only once it is fully incorporated, which this + * decides on its own: the sealed log covers generations up to + * new_live.start_generation - 1, so it is droppable exactly when + * first_unincorporated_generation >= new_live.start_generation. Otherwise it + * is preserved, because recovery would still need it. Callers therefore cannot + * drop a sealed log prematurely. */ void superblock_snapshot_tree(superblock_context *ctx, diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index 96f17054..ba5e7959 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -254,9 +254,13 @@ CTEST2(splinter, test_inserts) /* * With logging enabled, core_checkpoint() rotates the log via the two-log * protocol and advances the durable root. Data inserted before the checkpoint - * must survive it, a second (empty) checkpoint must be a clean rotate, and - * teardown's allocator_assert_noleaks() must pass (the sealed log's extents are - * freed, the root is not leaked or double-freed). + * must survive it, and teardown's allocator_assert_noleaks() must pass (the + * sealed log's extents are freed, the root is not leaked or double-freed). + * + * A zero rotation timeout forces the rotation immediately rather than waiting + * for insert traffic to trigger one. The second checkpoint takes no new + * inserts, so it also covers the empty-memtable forced rotation: the resulting + * generation retires with no branch at all. */ CTEST2(splinter, test_two_log_checkpoint) { @@ -281,7 +285,7 @@ CTEST2(splinter, test_two_log_checkpoint) // Checkpoint: seal the live log into the sealed slot, start a fresh live // log, incorporate, advance the durable root, then clear + free the sealed // log. - rc = core_checkpoint(&spl); + rc = core_checkpoint(&spl, 0); ASSERT_TRUE(SUCCESS(rc)); // A sample of keys must still be found after the checkpoint. @@ -306,7 +310,7 @@ CTEST2(splinter, test_two_log_checkpoint) lookup_result_deinit(&qdata); // Second checkpoint with no new inserts is a clean rotate. - rc = core_checkpoint(&spl); + rc = core_checkpoint(&spl, 0); ASSERT_TRUE(SUCCESS(rc)); core_destroy(&spl); diff --git a/tests/unit/superblock_test.c b/tests/unit/superblock_test.c index 9452602b..eb961ac2 100644 --- a/tests/unit/superblock_test.c +++ b/tests/unit/superblock_test.c @@ -155,13 +155,18 @@ CTEST2(superblock, test_snapshot_persists_state) superblock_context_deinit(&ctx); } -/* Steady, begin-checkpoint, and complete-checkpoint tree-record states. */ -static const superblock_log_head TEST_LOG_L1 = {.addr = 0x6000, - .meta_addr = 0x8000, - .magic = 0x11}; -static const superblock_log_head TEST_LOG_L2 = {.addr = 0x10000, - .meta_addr = 0x12000, - .magic = 0x22}; +/* + * Steady, begin-checkpoint, and complete-checkpoint tree-record states. L1 + * covers generations 0..5 and is cut at 5, so L2 takes over at 6. + */ +static const superblock_log_head TEST_LOG_L1 = {.addr = 0x6000, + .meta_addr = 0x8000, + .magic = 0x11, + .start_generation = 0}; +static const superblock_log_head TEST_LOG_L2 = {.addr = 0x10000, + .meta_addr = 0x12000, + .magic = 0x22, + .start_generation = 6}; /* * Walk the two-log checkpoint state machine through the superblock and confirm @@ -184,7 +189,7 @@ CTEST2(superblock, test_two_log_checkpoint_transitions) ASSERT_TRUE(SUCCESS(rc)); // Begin: cut the log -- L1 becomes sealed, L2 becomes live. - superblock_log_cut(&ctx, TEST_LOG_L2); + superblock_log_cut(&ctx, TEST_LOG_L1, TEST_LOG_L2); rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); @@ -193,7 +198,8 @@ CTEST2(superblock, test_two_log_checkpoint_transitions) ASSERT_EQUAL(TEST_LOG_L1.meta_addr, got.sealed_log.meta_addr); ASSERT_EQUAL(TEST_LOG_L2.meta_addr, got.live_log.meta_addr); - // Complete: advance root, carry L2 forward, clear sealed_log. + // Complete: advance the root past L1's coverage (first unincorporated 9 >= + // L2's start 6), so the sealed log is dropped and L2 carries forward. superblock_snapshot_tree(&ctx, 0x4400, 9, TEST_LOG_L2); rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); @@ -213,6 +219,52 @@ CTEST2(superblock, test_two_log_checkpoint_transitions) superblock_context_deinit(&ctx); } +/* + * A tree snapshot published while a sealed log is only partly incorporated must + * keep that sealed log: dropping it would strand the generations it still holds. + * + * This is the case a durability checkpoint hits when it commits a root that is + * newer than the sealed log's start but older than its end. L1 is sealed + * covering generations 0..5 (L2 starts at 6); a root whose first unincorporated + * generation is 5 has not yet folded in generation 5, so L1 must survive. Once + * a later root reaches 6, L1 is dropped. + */ +CTEST2(superblock, test_snapshot_preserves_unincorporated_sealed_log) +{ + superblock_context ctx; + platform_status rc = + superblock_context_init(&ctx, data->ioh, &data->allocator_cfg, data->hid); + ASSERT_TRUE(SUCCESS(rc)); + rc = superblock_format(&ctx, &data->allocator_cfg); + ASSERT_TRUE(SUCCESS(rc)); + + // A checkpoint is mid-flight: L1 sealed, L2 live. + superblock_log_cut(&ctx, TEST_LOG_L1, TEST_LOG_L2); + rc = superblock_make_durable(&ctx); + ASSERT_TRUE(SUCCESS(rc)); + + // Commit a root that stops short of L1's last generation: L1 must be kept. + superblock_snapshot_tree(&ctx, 0x4000, 5, TEST_LOG_L2); + rc = superblock_make_durable(&ctx); + ASSERT_TRUE(SUCCESS(rc)); + + superblock_tree_record got; + superblock_get_tree_record(&ctx, &got); + ASSERT_EQUAL(0x4000, got.root_addr); + ASSERT_EQUAL(TEST_LOG_L2.meta_addr, got.live_log.meta_addr); + ASSERT_EQUAL(TEST_LOG_L1.meta_addr, got.sealed_log.meta_addr); + ASSERT_EQUAL(TEST_LOG_L1.start_generation, got.sealed_log.start_generation); + + // Now a root that covers all of L1's generations: it is dropped. + superblock_snapshot_tree(&ctx, 0x4400, 6, TEST_LOG_L2); + rc = superblock_make_durable(&ctx); + ASSERT_TRUE(SUCCESS(rc)); + superblock_get_tree_record(&ctx, &got); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(got.sealed_log)); + + superblock_context_deinit(&ctx); +} + /* * A torn begin-checkpoint publish must leave the previous (steady) generation * intact: mount falls back to {root R0, live L1, no sealed}, which recovery can @@ -237,7 +289,7 @@ CTEST2(superblock, test_two_log_checkpoint_torn_begin) ASSERT_TRUE(SUCCESS(rc)); // Begin checkpoint: cut the log; this make_durable (gen5) targets slot0. - superblock_log_cut(&ctx, TEST_LOG_L2); + superblock_log_cut(&ctx, TEST_LOG_L1, TEST_LOG_L2); rc = superblock_make_durable(&ctx); // gen5 -> slot0 ASSERT_TRUE(SUCCESS(rc)); superblock_context_deinit(&ctx); From 38cb7e0abb4e0e8fab2f92f847919e38e8db37c2 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Wed, 29 Jul 2026 10:43:03 -0700 Subject: [PATCH 45/64] make core_checkpoint reclaim log space Signed-off-by: Rob Johnson --- src/core.c | 125 ++++++++++++++++++++++++++++--------- src/core.h | 17 ++--- tests/unit/splinter_test.c | 71 +++++++++++++++++++++ 3 files changed, 176 insertions(+), 37 deletions(-) diff --git a/src/core.c b/src/core.c index c44d1534..ae5f1973 100644 --- a/src/core.c +++ b/src/core.c @@ -358,7 +358,7 @@ core_checkpoint_commit_current_root(core_handle *spl, static bool32 core_should_take_checkpoint(core_handle *spl) { - if (!spl->cfg.use_log || spl->cfg.checkpoint_generation_interval == 0) { + if (spl->cfg.checkpoint_generation_interval == 0) { return FALSE; } uint64 current = spl->mt_ctxt.generation; @@ -366,17 +366,44 @@ core_should_take_checkpoint(core_handle *spl) >= spl->cfg.checkpoint_generation_interval; } +/* The current checkpoint phase. Read under the state lock. */ +static core_checkpoint_phase +core_checkpoint_phase_get(core_handle *spl) +{ + platform_mutex_lock(&spl->checkpoint_state_lock); + core_checkpoint_phase phase = spl->checkpoint.phase; + platform_mutex_unlock(&spl->checkpoint_state_lock); + return phase; +} + /* * Begin, step 1 (outside the rotation critical section): if no checkpoint is in - * progress and policy says so, pre-create the next live log and arm the swap. - * Log creation does no disk I/O, but is kept off the insert-blocking path. + * progress, and either the caller forces it or the interval policy says so, + * pre-create the next live log and arm the swap. Log creation does no disk I/O, + * but is kept off the insert-blocking path. + * + * `force` bypasses only the interval policy, never the use_log precondition: + * without a log there is nothing to cut, and arming would leave the rotate hook + * dereferencing a NULL spl->log. + * + * Sets *armed (if non-NULL) to whether this call armed the checkpoint, so a + * caller that wants to see its own checkpoint through can tell it did not merely + * observe someone else's. Declining because one is already in flight is not an + * error: only one checkpoint can be in flight at a time. */ static void -core_checkpoint_maybe_begin(core_handle *spl) +core_checkpoint_begin(core_handle *spl, bool32 force, bool32 *armed) { + if (armed != NULL) { + *armed = FALSE; + } + if (!spl->cfg.use_log) { + return; + } + platform_mutex_lock(&spl->checkpoint_state_lock); bool32 begin = spl->checkpoint.phase == CORE_CHECKPOINT_IDLE - && core_should_take_checkpoint(spl); + && (force || core_should_take_checkpoint(spl)); platform_mutex_unlock(&spl->checkpoint_state_lock); if (!begin) { return; @@ -386,7 +413,7 @@ core_checkpoint_maybe_begin(core_handle *spl) spl->cc, (shard_log_config *)spl->cfg.log_cfg, spl->heap_id); if (next == NULL) { platform_error_log( - "core_checkpoint_maybe_begin: shard_log_create failed; skipping\n"); + "core_checkpoint_begin: shard_log_create failed; skipping\n"); return; } log_head next_head = log_get_head(next); @@ -398,6 +425,9 @@ core_checkpoint_maybe_begin(core_handle *spl) spl->checkpoint.phase = CORE_CHECKPOINT_PENDING; spl->last_checkpoint_generation = spl->mt_ctxt.generation; next = NULL; // handed off to the checkpoint + if (armed != NULL) { + *armed = TRUE; + } } platform_mutex_unlock(&spl->checkpoint_state_lock); @@ -408,6 +438,13 @@ core_checkpoint_maybe_begin(core_handle *spl) } } +/* The automatic, policy-driven arm, run after every rotation. */ +static void +core_checkpoint_maybe_begin(core_handle *spl) +{ + core_checkpoint_begin(spl, FALSE /* force */, NULL); +} + /* * Begin, step 2 (inside the rotation critical section, insert lock held * exclusively): swap the pre-created live log in. Registered as @@ -2640,52 +2677,80 @@ core_seal_live_log(core_handle *spl) /* * Take a checkpoint: make every modification that completed before this call - * durable in the trunk root, and block until that is done. Safe to call on a - * running system with concurrent inserts. + * durable in the trunk root, reclaiming the retired log's space, and block until + * that is done. Safe to call on a running system with concurrent inserts. + * + * Durability reduces to a generation bound. Inserts land in the currently + * active memtable generation and generations only advance, so everything already + * inserted is in a generation at or below the one active at entry -- the target. + * Once the target is incorporated, all of it is in the trunk's COW root, and + * committing that root makes it durable. * - * The whole guarantee reduces to a generation bound. Inserts land in the - * currently active memtable generation and generations only advance, so - * everything already inserted is in a generation at or below the one active at - * entry -- call it the target. Once the target has been incorporated, every one - * of those inserts is in the trunk's COW root, and committing that root makes - * them durable. Log rotation is deliberately not part of this: the two-log - * protocol bounds replay and reclaims log extents, which is the automatic - * checkpoint policy's job, not a durability requirement. A checkpoint already - * in flight is therefore irrelevant here and is neither waited on nor disturbed. + * Reclaiming log space needs a log cut, which happens only when a rotation finds + * a checkpoint armed. So we arm one up front and then see it through: its + * completion frees the retired log's extents. This is what lets an application + * turn the interval policy off and manage log space entirely through this call. + * Arming is best effort -- only one checkpoint can be in flight, so if one + * already is we ride it out and leave reclamation to it. Durability does not + * depend on any of this. * - * The target is the *active* generation, so it cannot be incorporated until - * something finalizes it. Insert traffic normally does; if none arrives within - * rotation_timeout_ns we force the rotation. Forcing is safe even with an empty - * memtable: that generation retires with no branch at all (see - * core_memtable_compact()). + * Both halves wait on a rotation: the target cannot be incorporated until + * something finalizes it, and the cut cannot happen without one either. Insert + * traffic normally provides it; if none arrives within rotation_timeout_ns we + * force one. Forcing is safe even with an empty memtable -- that generation + * retires with no branch at all (see core_memtable_compact()) -- but note that a + * caller polling a quiescent database will force a rotation per call, spending a + * generation and a log extent each time. A longer timeout lets real traffic + * drive the cut instead. */ platform_status core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns) { uint64 target = memtable_generation(&spl->mt_ctxt); + bool32 armed; + core_checkpoint_begin(spl, TRUE /* force */, &armed); + uint64 wait = 100; timestamp deadline = platform_get_timestamp(); - while (memtable_generation_retired(&spl->mt_ctxt) + 1 <= target) { + while (TRUE) { + bool32 incorporated = + memtable_generation_retired(&spl->mt_ctxt) + 1 > target; + core_checkpoint_phase phase = core_checkpoint_phase_get(spl); /* - * Force a rotation only while the target is still the active generation. - * If it has already advanced, the target is finalized and merely needs its - * flush to drain -- forcing again would rotate an unrelated generation. + * Done once the target is durable-able and, if we started a checkpoint, + * it has completed (which is what freed the retired log). We only wait on + * a checkpoint we armed ourselves, so an unrelated one cannot hold us up. */ - if (memtable_generation(&spl->mt_ctxt) == target + if (incorporated && (!armed || phase == CORE_CHECKPOINT_IDLE)) { + break; + } + + /* + * Force a rotation if either half is still waiting on one: the target is + * still the active generation, or our checkpoint has yet to cut. Once + * neither holds, the remaining work is just draining flushes. + */ + bool32 needs_rotation = memtable_generation(&spl->mt_ctxt) == target + || (armed && phase == CORE_CHECKPOINT_PENDING); + if (needs_rotation && rotation_timeout_ns <= platform_timestamp_elapsed(deadline)) { memtable_force_rotation(&spl->mt_ctxt); // dispatches the flush itself + deadline = platform_get_timestamp(); } + task_perform_one_if_needed(spl->ts, 0); platform_sleep_ns(wait); wait = wait > 2048 ? wait : 2 * wait; } /* - * The target is incorporated. Commit the current root, recording the log - * that is actually live; superblock_snapshot_tree() keeps a sealed log that - * this root does not yet cover. + * Commit the current root, recording the log that is actually live. A + * completed checkpoint has already committed an equivalent root; republishing + * is harmless and this is still required when nothing was armed (or logging + * is off). superblock_snapshot_tree() keeps a sealed log this root does not + * yet cover. */ return core_checkpoint_commit_current_root(spl, core_current_live_log(spl)); } diff --git a/src/core.h b/src/core.h index c2b1428e..fb27aa05 100644 --- a/src/core.h +++ b/src/core.h @@ -315,14 +315,17 @@ core_mount(core_handle *spl, /* * Take a checkpoint: make every modification made before this call durable in - * the trunk root, rotating the log via the two-log protocol. Blocks until the - * checkpoint completes. Safe to call on a running system with concurrent - * inserts; if a checkpoint is already in flight, waits for it and then takes - * another (so the result always covers this caller's modifications). + * the trunk root, and reclaim the space of the log it retires. Blocks until + * both are done. Safe to call on a running system with concurrent inserts. * - * The log cut requires a memtable rotation, which insert traffic normally - * triggers. If none occurs within rotation_timeout_ns, the rotation is forced. - * Pass 0 to force it immediately. See core.c for the full contract. + * Reclamation makes this sufficient on its own, so an application can set + * checkpoint_generation_interval to 0 (no automatic checkpoints) and manage + * durability and log space entirely through this call. + * + * Both halves need a memtable rotation, which insert traffic normally triggers. + * If none occurs within rotation_timeout_ns, the rotation is forced; pass 0 to + * force it immediately. Beware that polling a quiescent database forces a + * rotation per call. See core.c for the full contract. */ platform_status core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns); diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index ba5e7959..f4a2d0cd 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -316,6 +316,77 @@ CTEST2(splinter, test_two_log_checkpoint) core_destroy(&spl); } +/* + * An application that manages checkpoints itself: the interval policy is off, so + * nothing arms a checkpoint automatically and core_checkpoint() is the only thing + * that can rotate the log. It must therefore cut the log and reclaim what it + * retires, or the live log would grow without bound. + * + * Each checkpoint must (a) install a different live log -- proving a cut + * happened -- and (b) leave the retired log's metadata extent free. Total space + * in use must also stay flat across many checkpoints of the same data. + */ +CTEST2(splinter, test_self_managed_checkpoints_reclaim_log_space) +{ + allocator *alp = (allocator *)&data->al; + data->system_cfg->splinter_cfg.use_log = TRUE; + // No automatic checkpoints: this test drives them all itself. + data->system_cfg->splinter_cfg.checkpoint_generation_interval = 0; + + core_handle spl; + platform_status rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + test_generate_allocator_root_id(), + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + uint64 num_inserts = splinter_do_inserts(data, &spl, FALSE, NULL); + ASSERT_NOT_EQUAL(0, num_inserts); + + superblock_tree_record rec; + const uint64 num_checkpoints = 10; + uint64 baseline_in_use = 0; + + for (uint64 i = 0; i < num_checkpoints; i++) { + superblock_get_tree_record(&spl.superblock, &rec); + uint64 retired_meta_addr = rec.live_log.meta_addr; + ASSERT_NOT_EQUAL(0, retired_meta_addr); + + rc = core_checkpoint(&spl, 0); + ASSERT_TRUE(SUCCESS(rc)); + + // (a) A cut happened: a different log is now live, and it covers only + // generations from the cut onward. + superblock_get_tree_record(&spl.superblock, &rec); + ASSERT_NOT_EQUAL(retired_meta_addr, rec.live_log.meta_addr); + ASSERT_TRUE(SUPERBLOCK_NO_LOG(rec.sealed_log)); + + // (b) The retired log's space came back. + ASSERT_EQUAL(0, + allocator_get_refcount(alp, retired_meta_addr), + "checkpoint %lu did not reclaim retired log at %lu\n", + i, + retired_meta_addr); + + // Space in use must not creep upward checkpoint over checkpoint. + if (i == 0) { + baseline_in_use = allocator_in_use(alp); + } else { + ASSERT_TRUE(allocator_in_use(alp) <= baseline_in_use, + "space in use grew from %lu to %lu by checkpoint %lu\n", + baseline_in_use, + allocator_in_use(alp), + i); + } + } + + core_destroy(&spl); +} + /* * Shared workload for the automatic-checkpoint tests: create with auto * checkpoints enabled, insert enough to drive several rotations, verify a From 9defdf631b8a6eff019540fc762cbc58851a8e6b Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Wed, 29 Jul 2026 10:43:16 -0700 Subject: [PATCH 46/64] formatting Signed-off-by: Rob Johnson --- src/core.c | 91 +++++++++++++++--------------- src/core.h | 5 +- src/superblock.c | 3 +- tests/functional/btree_test.c | 9 ++- tests/unit/splinter_test.c | 12 ++-- tests/unit/splinterdb_quick_test.c | 6 +- tests/unit/superblock_test.c | 6 +- 7 files changed, 71 insertions(+), 61 deletions(-) diff --git a/src/core.c b/src/core.c index ae5f1973..fde2e942 100644 --- a/src/core.c +++ b/src/core.c @@ -379,17 +379,17 @@ core_checkpoint_phase_get(core_handle *spl) /* * Begin, step 1 (outside the rotation critical section): if no checkpoint is in * progress, and either the caller forces it or the interval policy says so, - * pre-create the next live log and arm the swap. Log creation does no disk I/O, - * but is kept off the insert-blocking path. + * pre-create the next live log and arm the swap. Log creation does no disk + * I/O, but is kept off the insert-blocking path. * * `force` bypasses only the interval policy, never the use_log precondition: * without a log there is nothing to cut, and arming would leave the rotate hook * dereferencing a NULL spl->log. * * Sets *armed (if non-NULL) to whether this call armed the checkpoint, so a - * caller that wants to see its own checkpoint through can tell it did not merely - * observe someone else's. Declining because one is already in flight is not an - * error: only one checkpoint can be in flight at a time. + * caller that wants to see its own checkpoint through can tell it did not + * merely observe someone else's. Declining because one is already in flight is + * not an error: only one checkpoint can be in flight at a time. */ static void core_checkpoint_begin(core_handle *spl, bool32 force, bool32 *armed) @@ -482,10 +482,10 @@ core_rotate_log(void *arg, uint64 finalized_generation) * writers, so sealing is safe. * * Publishing here is what makes the cut crash-safe. The rotation moved inserts - * to the new live log, but the superblock still names the old one, so until this - * runs a crash would lose everything written to the new log. Order matters: the - * sealed log's pages must be durable before the superblock names it as sealed, - * since recovery replays it as-is. + * to the new live log, but the superblock still names the old one, so until + * this runs a crash would lose everything written to the new log. Order + * matters: the sealed log's pages must be durable before the superblock names + * it as sealed, since recovery replays it as-is. */ static void core_checkpoint_seal_cut(core_handle *spl) @@ -498,10 +498,10 @@ core_checkpoint_seal_cut(core_handle *spl) to_seal = spl->checkpoint.log_to_seal; spl->checkpoint.log_to_seal = NULL; spl->checkpoint.phase = CORE_CHECKPOINT_INCORPORATING; - sealed = core_log_to_superblock_log_head( + sealed = core_log_to_superblock_log_head( spl->checkpoint.sealed_head, spl->checkpoint.sealed_start_generation); live = core_log_to_superblock_log_head(spl->checkpoint.live_head, - spl->live_log_start_generation); + spl->live_log_start_generation); } platform_mutex_unlock(&spl->checkpoint_state_lock); @@ -893,12 +893,12 @@ core_memtable_compact(core_handle *spl, uint64 generation, const threadid tid) core_memtable_iterator_deinit(&btree_itor); /* - * A forced rotation (see memtable_force_rotation(), used by core_checkpoint() - * when nothing rotates on its own) can finalize a memtable that received no - * inserts. btree_pack() already defines this case: an empty input yields - * num_tuples == 0 and root_addr == 0, allocating no page. The generation - * still retires; core_memtable_incorporate() recognizes the missing branch - * and skips the trunk incorporation. + * A forced rotation (see memtable_force_rotation(), used by + * core_checkpoint() when nothing rotates on its own) can finalize a memtable + * that received no inserts. btree_pack() already defines this case: an + * empty input yields num_tuples == 0 and root_addr == 0, allocating no page. + * The generation still retires; core_memtable_incorporate() recognizes the + * missing branch and skips the trunk incorporation. */ new_branch->root_addr = req.root_addr; @@ -1000,8 +1000,8 @@ core_memtable_incorporate(core_handle *spl, */ bool32 has_branch = (cmt->branch.root_addr != 0); if (has_branch) { - rc = trunk_incorporate_prepare(&spl->trunk_context, - cmt->branch.root_addr); + rc = + trunk_incorporate_prepare(&spl->trunk_context, cmt->branch.root_addr); if (!SUCCESS(rc)) { platform_error_log("trunk_incorporate_prepare failed: %s\n", platform_status_to_string(rc)); @@ -2552,9 +2552,8 @@ core_mount(core_handle *spl, * session's log receives generations from the resume generation onward. */ spl->live_log_start_generation = resume_generation; - superblock_log_cut(&spl->superblock, - (superblock_log_head){0}, - core_current_live_log(spl)); + superblock_log_cut( + &spl->superblock, (superblock_log_head){0}, core_current_live_log(spl)); rc = superblock_make_durable(&spl->superblock); if (!SUCCESS(rc)) { platform_error_log("core_mount: mark-dirty superblock_make_durable " @@ -2677,31 +2676,32 @@ core_seal_live_log(core_handle *spl) /* * Take a checkpoint: make every modification that completed before this call - * durable in the trunk root, reclaiming the retired log's space, and block until - * that is done. Safe to call on a running system with concurrent inserts. + * durable in the trunk root, reclaiming the retired log's space, and block + * until that is done. Safe to call on a running system with concurrent + * inserts. * * Durability reduces to a generation bound. Inserts land in the currently - * active memtable generation and generations only advance, so everything already - * inserted is in a generation at or below the one active at entry -- the target. - * Once the target is incorporated, all of it is in the trunk's COW root, and - * committing that root makes it durable. + * active memtable generation and generations only advance, so everything + * already inserted is in a generation at or below the one active at entry -- + * the target. Once the target is incorporated, all of it is in the trunk's COW + * root, and committing that root makes it durable. * - * Reclaiming log space needs a log cut, which happens only when a rotation finds - * a checkpoint armed. So we arm one up front and then see it through: its - * completion frees the retired log's extents. This is what lets an application - * turn the interval policy off and manage log space entirely through this call. - * Arming is best effort -- only one checkpoint can be in flight, so if one - * already is we ride it out and leave reclamation to it. Durability does not - * depend on any of this. + * Reclaiming log space needs a log cut, which happens only when a rotation + * finds a checkpoint armed. So we arm one up front and then see it through: + * its completion frees the retired log's extents. This is what lets an + * application turn the interval policy off and manage log space entirely + * through this call. Arming is best effort -- only one checkpoint can be in + * flight, so if one already is we ride it out and leave reclamation to it. + * Durability does not depend on any of this. * * Both halves wait on a rotation: the target cannot be incorporated until * something finalizes it, and the cut cannot happen without one either. Insert * traffic normally provides it; if none arrives within rotation_timeout_ns we * force one. Forcing is safe even with an empty memtable -- that generation - * retires with no branch at all (see core_memtable_compact()) -- but note that a - * caller polling a quiescent database will force a rotation per call, spending a - * generation and a log extent each time. A longer timeout lets real traffic - * drive the cut instead. + * retires with no branch at all (see core_memtable_compact()) -- but note that + * a caller polling a quiescent database will force a rotation per call, + * spending a generation and a log extent each time. A longer timeout lets real + * traffic drive the cut instead. */ platform_status core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns) @@ -2719,8 +2719,9 @@ core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns) core_checkpoint_phase phase = core_checkpoint_phase_get(spl); /* * Done once the target is durable-able and, if we started a checkpoint, - * it has completed (which is what freed the retired log). We only wait on - * a checkpoint we armed ourselves, so an unrelated one cannot hold us up. + * it has completed (which is what freed the retired log). We only wait + * on a checkpoint we armed ourselves, so an unrelated one cannot hold us + * up. */ if (incorporated && (!armed || phase == CORE_CHECKPOINT_IDLE)) { break; @@ -2747,10 +2748,10 @@ core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns) /* * Commit the current root, recording the log that is actually live. A - * completed checkpoint has already committed an equivalent root; republishing - * is harmless and this is still required when nothing was armed (or logging - * is off). superblock_snapshot_tree() keeps a sealed log this root does not - * yet cover. + * completed checkpoint has already committed an equivalent root; + * republishing is harmless and this is still required when nothing was armed + * (or logging is off). superblock_snapshot_tree() keeps a sealed log this + * root does not yet cover. */ return core_checkpoint_commit_current_root(spl, core_current_live_log(spl)); } diff --git a/src/core.h b/src/core.h index fb27aa05..299de68b 100644 --- a/src/core.h +++ b/src/core.h @@ -189,8 +189,9 @@ struct core_handle { core_checkpoint_state checkpoint; uint64 last_checkpoint_generation; /* - * First memtable generation whose entries went to the current `log`. Recorded - * in the superblock so it can tell which generations each log covers. + * First memtable generation whose entries went to the current `log`. + * Recorded in the superblock so it can tell which generations each log + * covers. */ uint64 live_log_start_generation; diff --git a/src/superblock.c b/src/superblock.c index 06421520..87a496fa 100644 --- a/src/superblock.c +++ b/src/superblock.c @@ -236,7 +236,8 @@ superblock_snapshot_tree(superblock_context *ctx, uint64 first_unincorporated_generation, superblock_log_head new_live) { - // Advancing the root diverges the persisted allocation map, so invalidate it. + // Advancing the root diverges the persisted allocation map, so invalidate + // it. ctx->image->tree.root_addr = root_addr; ctx->image->tree.first_unincorporated_generation = first_unincorporated_generation; diff --git a/tests/functional/btree_test.c b/tests/functional/btree_test.c index d7a8c8f3..32b265e2 100644 --- a/tests/functional/btree_test.c +++ b/tests/functional/btree_test.c @@ -171,8 +171,13 @@ test_memtable_context_create(cache *cc, ctxt->cc = cc; ctxt->cfg = cfg; ctxt->heap_id = hid; - platform_status rc = memtable_context_init( - &ctxt->mt_ctxt, hid, cc, cfg->mt_cfg, NULL, test_btree_process_noop, NULL); + platform_status rc = memtable_context_init(&ctxt->mt_ctxt, + hid, + cc, + cfg->mt_cfg, + NULL, + test_btree_process_noop, + NULL); if (!SUCCESS(rc)) { platform_free(hid, ctxt); return NULL; diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index f4a2d0cd..ddef59f5 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -317,14 +317,14 @@ CTEST2(splinter, test_two_log_checkpoint) } /* - * An application that manages checkpoints itself: the interval policy is off, so - * nothing arms a checkpoint automatically and core_checkpoint() is the only thing - * that can rotate the log. It must therefore cut the log and reclaim what it - * retires, or the live log would grow without bound. + * An application that manages checkpoints itself: the interval policy is off, + * so nothing arms a checkpoint automatically and core_checkpoint() is the only + * thing that can rotate the log. It must therefore cut the log and reclaim + * what it retires, or the live log would grow without bound. * * Each checkpoint must (a) install a different live log -- proving a cut - * happened -- and (b) leave the retired log's metadata extent free. Total space - * in use must also stay flat across many checkpoints of the same data. + * happened -- and (b) leave the retired log's metadata extent free. Total + * space in use must also stay flat across many checkpoints of the same data. */ CTEST2(splinter, test_self_managed_checkpoints_reclaim_log_space) { diff --git a/tests/unit/splinterdb_quick_test.c b/tests/unit/splinterdb_quick_test.c index 09fa6ddc..4eb29e23 100644 --- a/tests/unit/splinterdb_quick_test.c +++ b/tests/unit/splinterdb_quick_test.c @@ -1809,9 +1809,9 @@ create_default_cfg(splinterdb_config *out_cfg, data_config *default_data_cfg) static uint64 force_flush_current_memtable(splinterdb *kvsb) { - core_handle *core = (core_handle *)splinterdb_get_trunk_handle(kvsb); - uint64 generation = memtable_force_rotation(&core->mt_ctxt); - platform_status rc = task_perform_until_quiescent(core->ts); + core_handle *core = (core_handle *)splinterdb_get_trunk_handle(kvsb); + uint64 generation = memtable_force_rotation(&core->mt_ctxt); + platform_status rc = task_perform_until_quiescent(core->ts); ASSERT_TRUE(SUCCESS(rc)); return generation; } diff --git a/tests/unit/superblock_test.c b/tests/unit/superblock_test.c index eb961ac2..93b2166f 100644 --- a/tests/unit/superblock_test.c +++ b/tests/unit/superblock_test.c @@ -96,7 +96,8 @@ CTEST2(superblock, test_format_sets_fresh_state) superblock_tree_record rec; superblock_get_tree_record(&ctx, &rec); ASSERT_EQUAL(0, rec.root_addr); // empty tree - ASSERT_EQUAL(0, rec.first_unincorporated_generation); // replay from the start + ASSERT_EQUAL(0, + rec.first_unincorporated_generation); // replay from the start superblock_context_deinit(&ctx); rc = @@ -221,7 +222,8 @@ CTEST2(superblock, test_two_log_checkpoint_transitions) /* * A tree snapshot published while a sealed log is only partly incorporated must - * keep that sealed log: dropping it would strand the generations it still holds. + * keep that sealed log: dropping it would strand the generations it still + * holds. * * This is the case a durability checkpoint hits when it commits a root that is * newer than the sealed log's start but older than its end. L1 is sealed From 40abda441bd292f2c62086c1fd24dab0f746d81f Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Wed, 29 Jul 2026 11:03:20 -0700 Subject: [PATCH 47/64] re-fix superblock api and callers Signed-off-by: Rob Johnson --- src/core.c | 139 +++++++++++++++++------------------ src/core.h | 22 ++---- src/superblock.c | 38 ++++++---- src/superblock.h | 43 ++++++----- tests/unit/superblock_test.c | 31 +++++--- 5 files changed, 145 insertions(+), 128 deletions(-) diff --git a/src/core.c b/src/core.c index fde2e942..907ac163 100644 --- a/src/core.c +++ b/src/core.c @@ -183,39 +183,23 @@ core_log_to_superblock_log_head(log_head info, uint64 start_generation) .start_generation = start_generation}; } -/* - * The superblock descriptor for the log currently receiving inserts. Read - * under checkpoint_state_lock, which core_rotate_log() holds while it swaps the - * live log, so this can never observe a half-swapped state (recording the - * just-sealed log as live would make recovery miss every post-cut insert). - */ -static superblock_log_head -core_current_live_log(core_handle *spl) -{ - if (!spl->cfg.use_log) { - return (superblock_log_head){0}; - } - platform_mutex_lock(&spl->checkpoint_state_lock); - superblock_log_head live = core_log_to_superblock_log_head( - log_get_head(spl->log), spl->live_log_start_generation); - platform_mutex_unlock(&spl->checkpoint_state_lock); - return live; -} - /* * Commit the trunk's current COW root as the new durable tree root: capture the * root, make its pages durable, snapshot it into the superblock (recording the - * first unincorporated generation, carrying the given live log forward, and - * clearing the sealed slot -- a snapshot happens only after the sealed log is - * folded in), then release the previously published root. snapshot_tree - * invalidates the persisted allocation state; a clean unmount revalidates it in - * a later step. Used by mkfs (empty root, records the new live log), - * checkpoint completion, and unmount (Part A). The snapshot's owned reference - * is transferred to the durable record on success. + * first unincorporated generation, and dropping the sealed log if this root now + * covers it), then release the previously published root. The log slots are the + * cut protocol's business, so this leaves the live log alone; callers that need + * to install or discard a log do so with superblock_log_cut() / + * superblock_discard_logs() before calling this, and the single publish below + * commits both transitions together. + * + * snapshot_tree invalidates the persisted allocation state; a clean unmount + * revalidates it in a later step. Used by mkfs, checkpoint completion, a + * durability checkpoint, and unmount (Part A). The snapshot's owned reference is + * transferred to the durable record on success. */ static platform_status -core_checkpoint_commit_current_root(core_handle *spl, - superblock_log_head live_log) +core_checkpoint_commit_current_root(core_handle *spl) { platform_status rc; trunk_snapshot snapshot; @@ -259,15 +243,13 @@ core_checkpoint_commit_current_root(core_handle *spl, old_root_addr = old_rec.root_addr; /* - * Snapshot the new root (clearing the sealed slot and carrying live_log - * forward) and make it durable. snapshot_tree invalidates the persisted - * allocation map -- the in-memory map now diverges from disk; a clean - * unmount revalidates it only after persisting the map (Part B). + * Snapshot the new root and make it durable. snapshot_tree invalidates the + * persisted allocation map -- the in-memory map now diverges from disk; a + * clean unmount revalidates it only after persisting the map (Part B). */ superblock_snapshot_tree(&spl->superblock, snapshot.root_addr, - first_unincorporated_generation, - live_log); + first_unincorporated_generation); rc = superblock_make_durable(&spl->superblock); if (!SUCCESS(rc)) { @@ -464,14 +446,15 @@ core_rotate_log(void *arg, uint64 finalized_generation) spl->log = spl->checkpoint.pending_log; spl->checkpoint.pending_log = NULL; /* - * Hand off generation coverage: the sealed log received everything up to - * and including finalized_generation, so the new live log starts at the - * next one. + * The retiring log received everything up to and including + * finalized_generation, so the new live log's coverage starts at the next + * one. The retiring log's own start generation needs no tracking here: + * the superblock already records it, and superblock_log_cut() carries it + * across into the sealed slot. */ - spl->checkpoint.sealed_start_generation = spl->live_log_start_generation; - spl->live_log_start_generation = finalized_generation + 1; - spl->checkpoint.cut_generation = finalized_generation; - spl->checkpoint.phase = CORE_CHECKPOINT_SEALING; + spl->checkpoint.live_start_generation = finalized_generation + 1; + spl->checkpoint.cut_generation = finalized_generation; + spl->checkpoint.phase = CORE_CHECKPOINT_SEALING; } platform_mutex_unlock(&spl->checkpoint_state_lock); } @@ -492,16 +475,13 @@ core_checkpoint_seal_cut(core_handle *spl) { platform_mutex_lock(&spl->checkpoint_state_lock); log_handle *to_seal = NULL; - superblock_log_head sealed = {0}; superblock_log_head live = {0}; if (spl->checkpoint.phase == CORE_CHECKPOINT_SEALING) { to_seal = spl->checkpoint.log_to_seal; spl->checkpoint.log_to_seal = NULL; spl->checkpoint.phase = CORE_CHECKPOINT_INCORPORATING; - sealed = core_log_to_superblock_log_head( - spl->checkpoint.sealed_head, spl->checkpoint.sealed_start_generation); - live = core_log_to_superblock_log_head(spl->checkpoint.live_head, - spl->live_log_start_generation); + live = core_log_to_superblock_log_head( + spl->checkpoint.live_head, spl->checkpoint.live_start_generation); } platform_mutex_unlock(&spl->checkpoint_state_lock); @@ -527,7 +507,9 @@ core_checkpoint_seal_cut(core_handle *spl) rc = cache_durable_barrier(spl->cc); } if (SUCCESS(rc)) { - superblock_log_cut(&spl->superblock, sealed, live); + // The image still names the retiring log as live, so the cut moves it into + // the sealed slot, carrying its recorded start generation with it. + superblock_log_cut(&spl->superblock, live); rc = superblock_make_durable(&spl->superblock); } if (!SUCCESS(rc)) { @@ -556,12 +538,9 @@ core_maybe_complete_checkpoint(core_handle *spl) uint64 first_unincorporated = memtable_generation_retired(&spl->mt_ctxt) + 1; bool32 complete = spl->checkpoint.phase == CORE_CHECKPOINT_INCORPORATING && first_unincorporated > spl->checkpoint.cut_generation; - log_head sealed = {0}; - superblock_log_head live = {0}; + log_head sealed = {0}; if (complete) { - sealed = spl->checkpoint.sealed_head; - live = core_log_to_superblock_log_head(spl->checkpoint.live_head, - spl->live_log_start_generation); + sealed = spl->checkpoint.sealed_head; spl->checkpoint.phase = CORE_CHECKPOINT_COMPLETING; } platform_mutex_unlock(&spl->checkpoint_state_lock); @@ -570,7 +549,12 @@ core_maybe_complete_checkpoint(core_handle *spl) return STATUS_OK; } - platform_status rc = core_checkpoint_commit_current_root(spl, live); + /* + * The live log is already recorded (core_checkpoint_seal_cut() published the + * cut), so this only advances the root -- which is what lets snapshot_tree + * drop the now-covered sealed log. + */ + platform_status rc = core_checkpoint_commit_current_root(spl); if (SUCCESS(rc)) { // The sealed log's entries are now durably in the root; free its extents. log_dec_ref(spl->cc, &sealed); @@ -2377,13 +2361,22 @@ core_mkfs(core_handle *spl, goto deinit_trunk_context; } - // Establish the initial (empty) tree record, recording the live log; publish - // it durably. No sealed log at mkfs; the log starts at generation 0. - spl->live_log_start_generation = 0; - rc = core_checkpoint_commit_current_root(spl, core_current_live_log(spl)); + /* + * Establish the initial (empty) tree record and install this instance's log, + * then publish both durably in one write. The formatted image has no live + * log, so the cut leaves the sealed slot empty; the log covers generations + * from 0. + */ + if (spl->cfg.use_log) { + superblock_log_cut( + &spl->superblock, + core_log_to_superblock_log_head(log_get_head(spl->log), 0)); + } + rc = core_checkpoint_commit_current_root(spl); if (!SUCCESS(rc)) { - platform_error_log("core_mkfs: core_commit_current_root failed: %s\n", - platform_status_to_string(rc)); + platform_error_log( + "core_mkfs: core_checkpoint_commit_current_root failed: %s\n", + platform_status_to_string(rc)); goto deinit_stats; } return STATUS_OK; @@ -2551,9 +2544,11 @@ core_mount(core_handle *spl, * mount has no prior live log, so the sealed slot stays empty. This * session's log receives generations from the resume generation onward. */ - spl->live_log_start_generation = resume_generation; - superblock_log_cut( - &spl->superblock, (superblock_log_head){0}, core_current_live_log(spl)); + superblock_log_cut(&spl->superblock, + spl->cfg.use_log + ? core_log_to_superblock_log_head( + log_get_head(spl->log), resume_generation) + : (superblock_log_head){0}); rc = superblock_make_durable(&spl->superblock); if (!SUCCESS(rc)) { platform_error_log("core_mount: mark-dirty superblock_make_durable " @@ -2747,13 +2742,14 @@ core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns) } /* - * Commit the current root, recording the log that is actually live. A - * completed checkpoint has already committed an equivalent root; - * republishing is harmless and this is still required when nothing was armed - * (or logging is off). superblock_snapshot_tree() keeps a sealed log this - * root does not yet cover. + * Advance the durable root. The log slots are left exactly as the cut + * protocol published them, so this cannot orphan a log; snapshot_tree only + * drops the sealed log if this root already covers it. A completed + * checkpoint has already committed an equivalent root -- republishing is + * harmless -- and this is still required when nothing was armed (or logging + * is off). */ - return core_checkpoint_commit_current_root(spl, core_current_live_log(spl)); + return core_checkpoint_commit_current_root(spl); } /* @@ -2784,10 +2780,13 @@ core_unmount(core_handle *spl) /* * Part A: publish the clean-unmount root with both log slots cleared (no - * live or sealed log at rest). Allocation state stays invalid here; it + * live or sealed log at rest -- their extents are freed just below, so no + * reference to them may survive). Both transitions go out in the single + * publish the commit performs. Allocation state stays invalid here; it * becomes valid only in Part B, after the map is persisted. */ - rc = core_checkpoint_commit_current_root(spl, (superblock_log_head){0}); + superblock_discard_logs(&spl->superblock); + rc = core_checkpoint_commit_current_root(spl); if (!SUCCESS(rc)) { platform_error_log("core_unmount: failed to publish unmount root: %s\n", platform_status_to_string(rc)); diff --git a/src/core.h b/src/core.h index 299de68b..e6be5bed 100644 --- a/src/core.h +++ b/src/core.h @@ -133,9 +133,10 @@ typedef struct core_checkpoint_state { log_handle *log_to_seal; // old live log awaiting seal (SEALING) log_head sealed_head; // identity of the sealed log (reclaim) log_head live_head; // identity of the new live log - // First generation the sealed log received; with live_log_start_generation - // this gives the sealed log's coverage for the superblock. - uint64 sealed_start_generation; + // First generation the new live log receives, recorded in the superblock as + // its coverage start. The retiring log's start needs no tracking: the + // superblock already holds it and carries it into the sealed slot. + uint64 live_start_generation; uint64 cut_generation; // complete once retired >= this } core_checkpoint_state; @@ -179,21 +180,14 @@ struct core_handle { /* * Incorporation-driven checkpoint state. checkpoint_state_lock guards the - * fields of `checkpoint`, last_checkpoint_generation, and - * live_log_start_generation (and is held while `log` is swapped); it is only - * ever held for brief, I/O-free updates (never across a barrier), so taking - * it inside the memtable rotation critical section cannot stall inserts on - * I/O. + * fields of `checkpoint` and last_checkpoint_generation (and is held while + * `log` is swapped); it is only ever held for brief, I/O-free updates (never + * across a barrier), so taking it inside the memtable rotation critical + * section cannot stall inserts on I/O. */ platform_mutex checkpoint_state_lock; core_checkpoint_state checkpoint; uint64 last_checkpoint_generation; - /* - * First memtable generation whose entries went to the current `log`. - * Recorded in the superblock so it can tell which generations each log - * covers. - */ - uint64 live_log_start_generation; core_stats *stats; diff --git a/src/superblock.c b/src/superblock.c index 87a496fa..09f88ed3 100644 --- a/src/superblock.c +++ b/src/superblock.c @@ -218,14 +218,13 @@ superblock_get_tree_record(const superblock_context *ctx, } void -superblock_log_cut(superblock_context *ctx, - superblock_log_head sealed, - superblock_log_head new_live) +superblock_log_cut(superblock_context *ctx, superblock_log_head new_live) { - // The sealed log's entries are still being folded into the next root; - // new_live receives subsequent inserts. The persisted allocation map no - // longer matches the (log) state, so invalidate it. - ctx->image->tree.sealed_log = sealed; + // The retiring live log becomes the sealed log: its entries are still being + // folded into the next root. new_live receives subsequent inserts. The + // persisted allocation map no longer matches the (log) state, so invalidate + // it. + ctx->image->tree.sealed_log = ctx->image->tree.live_log; ctx->image->tree.live_log = new_live; ctx->image->allocation_state_addr = 0; } @@ -233,30 +232,37 @@ superblock_log_cut(superblock_context *ctx, void superblock_snapshot_tree(superblock_context *ctx, uint64 root_addr, - uint64 first_unincorporated_generation, - superblock_log_head new_live) + uint64 first_unincorporated_generation) { // Advancing the root diverges the persisted allocation map, so invalidate - // it. + // it. The live log is left alone; see the header. ctx->image->tree.root_addr = root_addr; ctx->image->tree.first_unincorporated_generation = first_unincorporated_generation; - ctx->image->tree.live_log = new_live; ctx->image->allocation_state_addr = 0; /* * Drop the sealed log only once the new root covers everything it holds. It - * covers generations up to new_live.start_generation - 1, so it is fully + * covers generations up to live_log.start_generation - 1, so it is fully * incorporated exactly when first_unincorporated_generation reaches - * new_live.start_generation. Otherwise keep it: recovery still needs those - * entries. (A cleared live log means a clean shutdown, where everything is - * incorporated and start_generation is 0, so the sealed slot clears too.) + * live_log.start_generation. Otherwise keep it: recovery still needs those + * entries. */ - if (first_unincorporated_generation >= new_live.start_generation) { + if (first_unincorporated_generation + >= ctx->image->tree.live_log.start_generation) + { ctx->image->tree.sealed_log = (superblock_log_head){0}; } } +void +superblock_discard_logs(superblock_context *ctx) +{ + ctx->image->tree.sealed_log = (superblock_log_head){0}; + ctx->image->tree.live_log = (superblock_log_head){0}; + ctx->image->allocation_state_addr = 0; +} + void superblock_snapshot_allocator(superblock_context *ctx, uint64 map_addr) { diff --git a/src/superblock.h b/src/superblock.h index 1c4e5a79..b82b586a 100644 --- a/src/superblock.h +++ b/src/superblock.h @@ -189,38 +189,47 @@ superblock_format(superblock_context *ctx, const allocator_config *cfg); */ /* - * Record the log configuration produced by a cut: `sealed` is the just-retired - * log whose entries are still being folded into the next root (empty if there - * is none) and `new_live` receives subsequent inserts. Both slots are given - * explicitly rather than inferred from the current image, so this is correct - * regardless of what else has published in the meantime. Invalidates the - * allocation state. Used at a checkpoint's begin, and to install a fresh live - * log at mkfs/mount (with an empty sealed slot). + * Cut the log: the current live log becomes the sealed log -- its entries are + * still being folded into the next root -- and new_live receives subsequent + * inserts. Invalidates the allocation state. Used at a checkpoint's begin, and + * to install this session's log at mkfs/mount (where the current live log is + * empty, so the sealed slot stays empty). + * + * This is the only operation that installs a log, which is what lets it derive + * the sealed log from the image rather than taking it on trust. */ void -superblock_log_cut(superblock_context *ctx, - superblock_log_head sealed, - superblock_log_head new_live); +superblock_log_cut(superblock_context *ctx, superblock_log_head new_live); /* * Advance the durable tree to root_addr, where first_unincorporated_generation - * is the first generation not folded into it, carrying new_live forward as the - * live log (empty at a clean shutdown). Invalidates the allocation state. + * is the first generation not folded into it. Invalidates the allocation state. * Used at a checkpoint's completion, at a durability checkpoint, and at a clean * unmount. * + * Deliberately does not touch the live log: which log is live is the cut + * protocol's business (superblock_log_cut()), and a root advance that rewrote it + * could orphan a log still holding unincorporated entries. + * * The sealed log is dropped only once it is fully incorporated, which this * decides on its own: the sealed log covers generations up to - * new_live.start_generation - 1, so it is droppable exactly when - * first_unincorporated_generation >= new_live.start_generation. Otherwise it - * is preserved, because recovery would still need it. Callers therefore cannot + * live_log.start_generation - 1, so it is droppable exactly when + * first_unincorporated_generation >= live_log.start_generation. Otherwise it is + * preserved, because recovery would still need it. Callers therefore cannot * drop a sealed log prematurely. */ void superblock_snapshot_tree(superblock_context *ctx, uint64 root_addr, - uint64 first_unincorporated_generation, - superblock_log_head new_live); + uint64 first_unincorporated_generation); + +/* + * Drop both log slots. Used at a clean shutdown, which has folded everything + * into the durable root and frees the log's extents, so no log may remain + * referenced. Invalidates the allocation state. + */ +void +superblock_discard_logs(superblock_context *ctx); /* * Record the persisted allocator refcount map at map_addr as trustworthy. This diff --git a/tests/unit/superblock_test.c b/tests/unit/superblock_test.c index 93b2166f..b4f49324 100644 --- a/tests/unit/superblock_test.c +++ b/tests/unit/superblock_test.c @@ -127,7 +127,8 @@ CTEST2(superblock, test_snapshot_persists_state) superblock_log_head live = { .addr = 0x6000, .meta_addr = 0x8000, .magic = 0x11}; - superblock_snapshot_tree(&ctx, 0x4000, 0, live); + superblock_log_cut(&ctx, live); + superblock_snapshot_tree(&ctx, 0x4000, 0); rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); ASSERT_FALSE( @@ -185,12 +186,13 @@ CTEST2(superblock, test_two_log_checkpoint_transitions) ASSERT_TRUE(SUCCESS(rc)); // Steady: root R0, live L1, no sealed log. - superblock_snapshot_tree(&ctx, 0x4000, 5, TEST_LOG_L1); + superblock_log_cut(&ctx, TEST_LOG_L1); + superblock_snapshot_tree(&ctx, 0x4000, 5); rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); // Begin: cut the log -- L1 becomes sealed, L2 becomes live. - superblock_log_cut(&ctx, TEST_LOG_L1, TEST_LOG_L2); + superblock_log_cut(&ctx, TEST_LOG_L2); rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); @@ -201,7 +203,7 @@ CTEST2(superblock, test_two_log_checkpoint_transitions) // Complete: advance the root past L1's coverage (first unincorporated 9 >= // L2's start 6), so the sealed log is dropped and L2 carries forward. - superblock_snapshot_tree(&ctx, 0x4400, 9, TEST_LOG_L2); + superblock_snapshot_tree(&ctx, 0x4400, 9); rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); superblock_context_deinit(&ctx); @@ -240,13 +242,19 @@ CTEST2(superblock, test_snapshot_preserves_unincorporated_sealed_log) rc = superblock_format(&ctx, &data->allocator_cfg); ASSERT_TRUE(SUCCESS(rc)); - // A checkpoint is mid-flight: L1 sealed, L2 live. - superblock_log_cut(&ctx, TEST_LOG_L1, TEST_LOG_L2); + // Get to a checkpoint mid-flight: install L1, then cut to L2, which moves L1 + // into the sealed slot. + superblock_log_cut(&ctx, TEST_LOG_L1); + superblock_log_cut(&ctx, TEST_LOG_L2); rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); + superblock_tree_record mid; + superblock_get_tree_record(&ctx, &mid); + ASSERT_EQUAL(TEST_LOG_L1.meta_addr, mid.sealed_log.meta_addr); + ASSERT_EQUAL(TEST_LOG_L2.meta_addr, mid.live_log.meta_addr); // Commit a root that stops short of L1's last generation: L1 must be kept. - superblock_snapshot_tree(&ctx, 0x4000, 5, TEST_LOG_L2); + superblock_snapshot_tree(&ctx, 0x4000, 5); rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); @@ -258,7 +266,7 @@ CTEST2(superblock, test_snapshot_preserves_unincorporated_sealed_log) ASSERT_EQUAL(TEST_LOG_L1.start_generation, got.sealed_log.start_generation); // Now a root that covers all of L1's generations: it is dropped. - superblock_snapshot_tree(&ctx, 0x4400, 6, TEST_LOG_L2); + superblock_snapshot_tree(&ctx, 0x4400, 6); rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); superblock_get_tree_record(&ctx, &got); @@ -284,14 +292,15 @@ CTEST2(superblock, test_two_log_checkpoint_torn_begin) // Make the steady state durable into BOTH slots (gen3->slot0, gen4->slot1), // so the fallback below is unambiguously the steady state, not the empty // format one. - superblock_snapshot_tree(&ctx, 0x4000, 5, TEST_LOG_L1); + superblock_log_cut(&ctx, TEST_LOG_L1); + superblock_snapshot_tree(&ctx, 0x4000, 5); rc = superblock_make_durable(&ctx); // gen3 -> slot0 ASSERT_TRUE(SUCCESS(rc)); rc = superblock_make_durable(&ctx); // gen4 -> slot1 ASSERT_TRUE(SUCCESS(rc)); // Begin checkpoint: cut the log; this make_durable (gen5) targets slot0. - superblock_log_cut(&ctx, TEST_LOG_L1, TEST_LOG_L2); + superblock_log_cut(&ctx, TEST_LOG_L2); rc = superblock_make_durable(&ctx); // gen5 -> slot0 ASSERT_TRUE(SUCCESS(rc)); superblock_context_deinit(&ctx); @@ -330,7 +339,7 @@ CTEST2(superblock, test_torn_write_falls_back_to_older_generation) // After format the image is gen 2 in slot 1, so this make_durable targets // slot 0. Snapshot a nonempty root with no live log. - superblock_snapshot_tree(&ctx, 0x4000, 0, (superblock_log_head){0}); + superblock_snapshot_tree(&ctx, 0x4000, 0); rc = superblock_make_durable(&ctx); ASSERT_TRUE(SUCCESS(rc)); superblock_context_deinit(&ctx); From 4154de35221184b66b5c66e8cf7e231112502cf1 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Wed, 29 Jul 2026 11:03:27 -0700 Subject: [PATCH 48/64] formatting Signed-off-by: Rob Johnson --- src/core.c | 25 ++++++++++++------------- src/superblock.h | 20 ++++++++++---------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/core.c b/src/core.c index 907ac163..377876f6 100644 --- a/src/core.c +++ b/src/core.c @@ -187,16 +187,16 @@ core_log_to_superblock_log_head(log_head info, uint64 start_generation) * Commit the trunk's current COW root as the new durable tree root: capture the * root, make its pages durable, snapshot it into the superblock (recording the * first unincorporated generation, and dropping the sealed log if this root now - * covers it), then release the previously published root. The log slots are the - * cut protocol's business, so this leaves the live log alone; callers that need - * to install or discard a log do so with superblock_log_cut() / + * covers it), then release the previously published root. The log slots are + * the cut protocol's business, so this leaves the live log alone; callers that + * need to install or discard a log do so with superblock_log_cut() / * superblock_discard_logs() before calling this, and the single publish below * commits both transitions together. * * snapshot_tree invalidates the persisted allocation state; a clean unmount * revalidates it in a later step. Used by mkfs, checkpoint completion, a - * durability checkpoint, and unmount (Part A). The snapshot's owned reference is - * transferred to the durable record on success. + * durability checkpoint, and unmount (Part A). The snapshot's owned reference + * is transferred to the durable record on success. */ static platform_status core_checkpoint_commit_current_root(core_handle *spl) @@ -247,9 +247,8 @@ core_checkpoint_commit_current_root(core_handle *spl) * persisted allocation map -- the in-memory map now diverges from disk; a * clean unmount revalidates it only after persisting the map (Part B). */ - superblock_snapshot_tree(&spl->superblock, - snapshot.root_addr, - first_unincorporated_generation); + superblock_snapshot_tree( + &spl->superblock, snapshot.root_addr, first_unincorporated_generation); rc = superblock_make_durable(&spl->superblock); if (!SUCCESS(rc)) { @@ -453,8 +452,8 @@ core_rotate_log(void *arg, uint64 finalized_generation) * across into the sealed slot. */ spl->checkpoint.live_start_generation = finalized_generation + 1; - spl->checkpoint.cut_generation = finalized_generation; - spl->checkpoint.phase = CORE_CHECKPOINT_SEALING; + spl->checkpoint.cut_generation = finalized_generation; + spl->checkpoint.phase = CORE_CHECKPOINT_SEALING; } platform_mutex_unlock(&spl->checkpoint_state_lock); } @@ -480,7 +479,7 @@ core_checkpoint_seal_cut(core_handle *spl) to_seal = spl->checkpoint.log_to_seal; spl->checkpoint.log_to_seal = NULL; spl->checkpoint.phase = CORE_CHECKPOINT_INCORPORATING; - live = core_log_to_superblock_log_head( + live = core_log_to_superblock_log_head( spl->checkpoint.live_head, spl->checkpoint.live_start_generation); } platform_mutex_unlock(&spl->checkpoint_state_lock); @@ -507,8 +506,8 @@ core_checkpoint_seal_cut(core_handle *spl) rc = cache_durable_barrier(spl->cc); } if (SUCCESS(rc)) { - // The image still names the retiring log as live, so the cut moves it into - // the sealed slot, carrying its recorded start generation with it. + // The image still names the retiring log as live, so the cut moves it + // into the sealed slot, carrying its recorded start generation with it. superblock_log_cut(&spl->superblock, live); rc = superblock_make_durable(&spl->superblock); } diff --git a/src/superblock.h b/src/superblock.h index b82b586a..16922a32 100644 --- a/src/superblock.h +++ b/src/superblock.h @@ -191,9 +191,9 @@ superblock_format(superblock_context *ctx, const allocator_config *cfg); /* * Cut the log: the current live log becomes the sealed log -- its entries are * still being folded into the next root -- and new_live receives subsequent - * inserts. Invalidates the allocation state. Used at a checkpoint's begin, and - * to install this session's log at mkfs/mount (where the current live log is - * empty, so the sealed slot stays empty). + * inserts. Invalidates the allocation state. Used at a checkpoint's begin, + * and to install this session's log at mkfs/mount (where the current live log + * is empty, so the sealed slot stays empty). * * This is the only operation that installs a log, which is what lets it derive * the sealed log from the image rather than taking it on trust. @@ -203,19 +203,19 @@ superblock_log_cut(superblock_context *ctx, superblock_log_head new_live); /* * Advance the durable tree to root_addr, where first_unincorporated_generation - * is the first generation not folded into it. Invalidates the allocation state. - * Used at a checkpoint's completion, at a durability checkpoint, and at a clean - * unmount. + * is the first generation not folded into it. Invalidates the allocation + * state. Used at a checkpoint's completion, at a durability checkpoint, and at + * a clean unmount. * * Deliberately does not touch the live log: which log is live is the cut - * protocol's business (superblock_log_cut()), and a root advance that rewrote it - * could orphan a log still holding unincorporated entries. + * protocol's business (superblock_log_cut()), and a root advance that rewrote + * it could orphan a log still holding unincorporated entries. * * The sealed log is dropped only once it is fully incorporated, which this * decides on its own: the sealed log covers generations up to * live_log.start_generation - 1, so it is droppable exactly when - * first_unincorporated_generation >= live_log.start_generation. Otherwise it is - * preserved, because recovery would still need it. Callers therefore cannot + * first_unincorporated_generation >= live_log.start_generation. Otherwise it + * is preserved, because recovery would still need it. Callers therefore cannot * drop a sealed log prematurely. */ void From a05fd106423d3b1cafbcc268f8f8b3877c06bbfc Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Wed, 29 Jul 2026 11:53:05 -0700 Subject: [PATCH 49/64] renaming Signed-off-by: Rob Johnson --- src/core.c | 14 +++++++------- src/core.h | 5 ++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/core.c b/src/core.c index 377876f6..8691d2ee 100644 --- a/src/core.c +++ b/src/core.c @@ -51,7 +51,7 @@ static platform_status core_checkpoint_lock_init(core_handle *spl) { platform_status rc = platform_mutex_init( - &spl->checkpoint_lock, platform_get_module_id(), spl->heap_id); + &spl->superblock_lock, platform_get_module_id(), spl->heap_id); if (!SUCCESS(rc)) { return rc; } @@ -60,7 +60,7 @@ core_checkpoint_lock_init(core_handle *spl) &spl->checkpoint_state_lock, platform_get_module_id(), spl->heap_id); if (!SUCCESS(rc)) { platform_status destroy_rc = - platform_mutex_destroy(&spl->checkpoint_lock); + platform_mutex_destroy(&spl->superblock_lock); platform_assert_status_ok(destroy_rc); return rc; } @@ -75,7 +75,7 @@ core_checkpoint_lock_deinit(core_handle *spl) { platform_status rc = platform_mutex_destroy(&spl->checkpoint_state_lock); platform_assert_status_ok(rc); - rc = platform_mutex_destroy(&spl->checkpoint_lock); + rc = platform_mutex_destroy(&spl->superblock_lock); platform_assert_status_ok(rc); } @@ -211,7 +211,7 @@ core_checkpoint_commit_current_root(core_handle *spl) * The snapshot cut, durable record write, and old-root release are one * publication transaction; serialize against any concurrent publisher. */ - rc = platform_mutex_lock(&spl->checkpoint_lock); + rc = platform_mutex_lock(&spl->superblock_lock); if (!SUCCESS(rc)) { return rc; } @@ -309,7 +309,7 @@ core_checkpoint_commit_current_root(core_handle *spl) unlock_checkpoint: { - platform_status unlock_rc = platform_mutex_unlock(&spl->checkpoint_lock); + platform_status unlock_rc = platform_mutex_unlock(&spl->superblock_lock); if (SUCCESS(rc) && !SUCCESS(unlock_rc)) { rc = unlock_rc; } @@ -495,7 +495,7 @@ core_checkpoint_seal_cut(core_handle *spl) * completion publish will record the correct final state; only this * crash-protection window is left uncovered. */ - platform_status rc = platform_mutex_lock(&spl->checkpoint_lock); + platform_status rc = platform_mutex_lock(&spl->superblock_lock); if (!SUCCESS(rc)) { platform_error_log("core_checkpoint_seal_cut: lock failed: %s\n", platform_status_to_string(rc)); @@ -516,7 +516,7 @@ core_checkpoint_seal_cut(core_handle *spl) "cut: %s\n", platform_status_to_string(rc)); } - platform_status unlock_rc = platform_mutex_unlock(&spl->checkpoint_lock); + platform_status unlock_rc = platform_mutex_unlock(&spl->superblock_lock); platform_assert_status_ok(unlock_rc); } diff --git a/src/core.h b/src/core.h index e6be5bed..68942ad3 100644 --- a/src/core.h +++ b/src/core.h @@ -173,10 +173,9 @@ struct core_handle { * when multi-tree support lands this ownership hoists to an instance level * that per-tree cores borrow. */ - superblock_context superblock; - /* Serializes snapshot cuts and superblock publication. */ - platform_mutex checkpoint_lock; + platform_mutex superblock_lock; + superblock_context superblock; /* * Incorporation-driven checkpoint state. checkpoint_state_lock guards the From bd888f83dcff763dedc3fd148cae61159aa612b1 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Wed, 29 Jul 2026 12:04:01 -0700 Subject: [PATCH 50/64] simplify commit_current_root Signed-off-by: Rob Johnson --- src/core.c | 108 ++++++++++++++++++++++++++--------------------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/src/core.c b/src/core.c index 8691d2ee..e0aa15b4 100644 --- a/src/core.c +++ b/src/core.c @@ -47,8 +47,14 @@ static const int64 latency_histo_buckets[LATENCYHISTO_SIZE] = { _Static_assert(CORE_NUM_MEMTABLES <= MAX_MEMTABLES, "CORE_NUM_MEMTABLES <= MAX_MEMTABLES"); +/* + * Initialize the instance's two locks -- superblock_lock (publication) and + * checkpoint_state_lock (the checkpoint phase machine) -- plus the checkpoint + * state they guard. Transactional: on failure nothing is left initialized, so + * callers just bail out and must not call core_locks_deinit(). + */ static platform_status -core_checkpoint_lock_init(core_handle *spl) +core_locks_init(core_handle *spl) { platform_status rc = platform_mutex_init( &spl->superblock_lock, platform_get_module_id(), spl->heap_id); @@ -71,7 +77,7 @@ core_checkpoint_lock_init(core_handle *spl) } static void -core_checkpoint_lock_deinit(core_handle *spl) +core_locks_deinit(core_handle *spl) { platform_status rc = platform_mutex_destroy(&spl->checkpoint_state_lock); platform_assert_status_ok(rc); @@ -195,8 +201,15 @@ core_log_to_superblock_log_head(log_head info, uint64 start_generation) * * snapshot_tree invalidates the persisted allocation state; a clean unmount * revalidates it in a later step. Used by mkfs, checkpoint completion, a - * durability checkpoint, and unmount (Part A). The snapshot's owned reference - * is transferred to the durable record on success. + * durability checkpoint, and unmount (Part A). On success the captured + * reference becomes the durable record's and the previously published root's is + * released; republishing an unchanged root is just the degenerate case of that, + * so it needs no special handling. + * + * Note this always publishes, even when the root is unchanged: callers stage log + * transitions into the image beforehand, and the generation bound can advance on + * its own (an empty generation retires without changing the root), so the root + * address alone is not a "nothing to do" test. */ static platform_status core_checkpoint_commit_current_root(core_handle *spl) @@ -219,7 +232,7 @@ core_checkpoint_commit_current_root(core_handle *spl) rc = core_checkpoint_capture_cut( spl, &snapshot, &first_unincorporated_generation); if (!SUCCESS(rc)) { - goto unlock_checkpoint; + goto unlock_superblock; } /* @@ -256,47 +269,34 @@ core_checkpoint_commit_current_root(core_handle *spl) goto release_snapshot; } - if (old_root_addr == snapshot.root_addr) { - /* - * Republishing the same root (e.g. a clean unmount with no advance since - * the last publish): the previously published record already accounts - * for a reference to it, so drop the redundant snapshot reference we - * captured rather than letting the root's refcount grow each cycle. - */ - if (snapshot.root_addr != 0) { - platform_status release_rc = - trunk_snapshot_release(&spl->trunk_context, &snapshot); - if (SUCCESS(rc) && !SUCCESS(release_rc)) { + /* + * The captured reference becomes the record's durable one, and the previously + * published root's reference is released. This is uniform even when the root + * did not change: that root's count is momentarily 2 (the record's plus ours) + * and the release brings it back to the record's single reference, so a + * same-root republish needs no special case and cannot grow the count. The + * publish barrier already committed the new root to the newer superblock slot, + * so the old slot is no longer the mount choice and releasing it cannot + * strand a torn-write fallback. + */ + snapshot.root_addr = 0; // transferred to the durable record + if (old_root_addr != 0) { + trunk_snapshot old_snapshot = {.root_addr = old_root_addr}; + platform_status release_rc = + trunk_snapshot_release(&spl->trunk_context, &old_snapshot); + if (!SUCCESS(release_rc)) { + platform_error_log("core_checkpoint_commit_current_root: " + "trunk_snapshot_release failed for old root addr " + "%lu: %s\n", + old_root_addr, + platform_status_to_string(release_rc)); + if (SUCCESS(rc)) { rc = release_rc; } } - } else { - /* - * New root: the snapshot reference becomes the record's durable one, and - * the previously published root's reference is released. The publish - * barrier committed the new root to the newer superblock slot, so the - * old slot is no longer the mount choice and releasing it cannot strand a - * torn-write fallback. - */ - snapshot.root_addr = 0; // transferred to the durable record - if (old_root_addr != 0) { - trunk_snapshot old_snapshot = {.root_addr = old_root_addr}; - platform_status release_rc = - trunk_snapshot_release(&spl->trunk_context, &old_snapshot); - if (!SUCCESS(release_rc)) { - platform_error_log( - "core_commit_current_root: trunk_snapshot_release " - "failed for old root addr %lu: %s\n", - old_root_addr, - platform_status_to_string(release_rc)); - if (SUCCESS(rc)) { - rc = release_rc; - } - } - } } - goto unlock_checkpoint; + goto unlock_superblock; release_snapshot: { @@ -307,7 +307,7 @@ core_checkpoint_commit_current_root(core_handle *spl) } } -unlock_checkpoint: +unlock_superblock: { platform_status unlock_rc = platform_mutex_unlock(&spl->superblock_lock); if (SUCCESS(rc) && !SUCCESS(unlock_rc)) { @@ -2291,10 +2291,10 @@ core_mkfs(core_handle *spl, spl->heap_id = hid; spl->ts = ts; - platform_status rc = core_checkpoint_lock_init(spl); + platform_status rc = core_locks_init(spl); if (!SUCCESS(rc)) { platform_error_log( - "core_mkfs: checkpoint lock initialization failed: %s\n", + "core_mkfs: lock initialization failed: %s\n", platform_status_to_string(rc)); return rc; } @@ -2305,7 +2305,7 @@ core_mkfs(core_handle *spl, if (!SUCCESS(rc)) { platform_error_log("core_mkfs: superblock_context_init failed: %s\n", platform_status_to_string(rc)); - goto deinit_checkpoint_lock; + goto deinit_locks; } rc = superblock_format(&spl->superblock, allocator_cfg); if (!SUCCESS(rc)) { @@ -2393,8 +2393,8 @@ core_mkfs(core_handle *spl, memtable_context_deinit(&spl->mt_ctxt); deinit_superblock: superblock_context_deinit(&spl->superblock); -deinit_checkpoint_lock: - core_checkpoint_lock_deinit(spl); +deinit_locks: + core_locks_deinit(spl); return rc; } @@ -2421,10 +2421,10 @@ core_mount(core_handle *spl, spl->heap_id = hid; spl->ts = ts; - platform_status rc = core_checkpoint_lock_init(spl); + platform_status rc = core_locks_init(spl); if (!SUCCESS(rc)) { platform_error_log( - "core_mount: checkpoint lock initialization failed: %s\n", + "core_mount: lock initialization failed: %s\n", platform_status_to_string(rc)); return rc; } @@ -2435,7 +2435,7 @@ core_mount(core_handle *spl, if (!SUCCESS(rc)) { platform_error_log("core_mount: superblock_context_init failed: %s\n", platform_status_to_string(rc)); - goto deinit_checkpoint_lock; + goto deinit_locks; } rc = superblock_mount(&spl->superblock, allocator_cfg); if (!SUCCESS(rc)) { @@ -2570,8 +2570,8 @@ core_mount(core_handle *spl, memtable_context_deinit(&spl->mt_ctxt); deinit_superblock: superblock_context_deinit(&spl->superblock); -deinit_checkpoint_lock: - core_checkpoint_lock_deinit(spl); +deinit_locks: + core_locks_deinit(spl); return rc; } @@ -2830,7 +2830,7 @@ core_unmount(core_handle *spl) superblock_context_deinit(&spl->superblock); core_destroy_stats(spl); - core_checkpoint_lock_deinit(spl); + core_locks_deinit(spl); return rc; } @@ -2882,7 +2882,7 @@ core_destroy(core_handle *spl) */ superblock_context_deinit(&spl->superblock); core_destroy_stats(spl); - core_checkpoint_lock_deinit(spl); + core_locks_deinit(spl); } From 332421627cc663c1d2ff12e9d87f8ce50c16f130 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Wed, 29 Jul 2026 12:10:08 -0700 Subject: [PATCH 51/64] make core_checkpoint_begin return bool Signed-off-by: Rob Johnson --- src/core.c | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/src/core.c b/src/core.c index e0aa15b4..386668c6 100644 --- a/src/core.c +++ b/src/core.c @@ -367,19 +367,16 @@ core_checkpoint_phase_get(core_handle *spl) * without a log there is nothing to cut, and arming would leave the rotate hook * dereferencing a NULL spl->log. * - * Sets *armed (if non-NULL) to whether this call armed the checkpoint, so a - * caller that wants to see its own checkpoint through can tell it did not - * merely observe someone else's. Declining because one is already in flight is - * not an error: only one checkpoint can be in flight at a time. + * Returns whether this call is what armed the checkpoint, so a caller that wants + * to see its own checkpoint through can tell it did not merely observe someone + * else's. Returning FALSE is not an error: only one checkpoint can be in flight + * at a time, and declining is the normal outcome when one already is. */ -static void -core_checkpoint_begin(core_handle *spl, bool32 force, bool32 *armed) +static bool32 +core_checkpoint_begin(core_handle *spl, bool32 force) { - if (armed != NULL) { - *armed = FALSE; - } if (!spl->cfg.use_log) { - return; + return FALSE; } platform_mutex_lock(&spl->checkpoint_state_lock); @@ -387,7 +384,7 @@ core_checkpoint_begin(core_handle *spl, bool32 force, bool32 *armed) && (force || core_should_take_checkpoint(spl)); platform_mutex_unlock(&spl->checkpoint_state_lock); if (!begin) { - return; + return FALSE; } log_handle *next = shard_log_create( @@ -395,10 +392,11 @@ core_checkpoint_begin(core_handle *spl, bool32 force, bool32 *armed) if (next == NULL) { platform_error_log( "core_checkpoint_begin: shard_log_create failed; skipping\n"); - return; + return FALSE; } log_head next_head = log_get_head(next); + bool32 armed = FALSE; platform_mutex_lock(&spl->checkpoint_state_lock); if (spl->checkpoint.phase == CORE_CHECKPOINT_IDLE) { spl->checkpoint.pending_log = next; @@ -406,9 +404,7 @@ core_checkpoint_begin(core_handle *spl, bool32 force, bool32 *armed) spl->checkpoint.phase = CORE_CHECKPOINT_PENDING; spl->last_checkpoint_generation = spl->mt_ctxt.generation; next = NULL; // handed off to the checkpoint - if (armed != NULL) { - *armed = TRUE; - } + armed = TRUE; } platform_mutex_unlock(&spl->checkpoint_state_lock); @@ -417,13 +413,14 @@ core_checkpoint_begin(core_handle *spl, bool32 force, bool32 *armed) log_seal(next); log_dec_ref(spl->cc, &next_head); } + return armed; } /* The automatic, policy-driven arm, run after every rotation. */ static void core_checkpoint_maybe_begin(core_handle *spl) { - core_checkpoint_begin(spl, FALSE /* force */, NULL); + core_checkpoint_begin(spl, FALSE /* force */); } /* @@ -2702,8 +2699,7 @@ core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns) { uint64 target = memtable_generation(&spl->mt_ctxt); - bool32 armed; - core_checkpoint_begin(spl, TRUE /* force */, &armed); + bool32 armed = core_checkpoint_begin(spl, TRUE /* force */); uint64 wait = 100; timestamp deadline = platform_get_timestamp(); From 60735e8ce16dca9b7e9472a5bb969c4f8c3661ed Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Wed, 29 Jul 2026 13:48:30 -0700 Subject: [PATCH 52/64] avoid excess waiting in core_checkpoint Signed-off-by: Rob Johnson --- src/core.c | 72 +++++++++++++++++++++++++++++++++++------------------- src/core.h | 10 ++++++++ 2 files changed, 57 insertions(+), 25 deletions(-) diff --git a/src/core.c b/src/core.c index 386668c6..6fea2845 100644 --- a/src/core.c +++ b/src/core.c @@ -347,14 +347,24 @@ core_should_take_checkpoint(core_handle *spl) >= spl->cfg.checkpoint_generation_interval; } -/* The current checkpoint phase. Read under the state lock. */ -static core_checkpoint_phase -core_checkpoint_phase_get(core_handle *spl) +/* + * A consistent read of what a waiter needs to poll: the current phase (has the + * cut happened yet?) and the completion count (has a given checkpoint finished + * and freed its log?). Taken together under one acquisition of the state lock. + */ +typedef struct core_checkpoint_status { + core_checkpoint_phase phase; + uint64 completions; +} core_checkpoint_status; + +static core_checkpoint_status +core_checkpoint_status_get(core_handle *spl) { platform_mutex_lock(&spl->checkpoint_state_lock); - core_checkpoint_phase phase = spl->checkpoint.phase; + core_checkpoint_status status = {.phase = spl->checkpoint.phase, + .completions = spl->checkpoint.completions}; platform_mutex_unlock(&spl->checkpoint_state_lock); - return phase; + return status; } /* @@ -367,16 +377,19 @@ core_checkpoint_phase_get(core_handle *spl) * without a log there is nothing to cut, and arming would leave the rotate hook * dereferencing a NULL spl->log. * - * Returns whether this call is what armed the checkpoint, so a caller that wants - * to see its own checkpoint through can tell it did not merely observe someone - * else's. Returning FALSE is not an error: only one checkpoint can be in flight - * at a time, and declining is the normal outcome when one already is. + * Returns a completion ticket for the checkpoint this call armed: it has + * finished, and freed its retired log, once checkpoint.completions reaches the + * ticket. The ticket is captured under the same lock acquisition that arms, so + * it cannot miss or over-count a completion. Returns 0 if this call did not arm + * one, which is not an error -- only one checkpoint can be in flight at a time, + * and declining is the normal outcome when one already is. Tickets are 1-based, + * so 0 is unambiguous. */ -static bool32 +static uint64 core_checkpoint_begin(core_handle *spl, bool32 force) { if (!spl->cfg.use_log) { - return FALSE; + return 0; } platform_mutex_lock(&spl->checkpoint_state_lock); @@ -384,7 +397,7 @@ core_checkpoint_begin(core_handle *spl, bool32 force) && (force || core_should_take_checkpoint(spl)); platform_mutex_unlock(&spl->checkpoint_state_lock); if (!begin) { - return FALSE; + return 0; } log_handle *next = shard_log_create( @@ -392,11 +405,11 @@ core_checkpoint_begin(core_handle *spl, bool32 force) if (next == NULL) { platform_error_log( "core_checkpoint_begin: shard_log_create failed; skipping\n"); - return FALSE; + return 0; } log_head next_head = log_get_head(next); - bool32 armed = FALSE; + uint64 ticket = 0; platform_mutex_lock(&spl->checkpoint_state_lock); if (spl->checkpoint.phase == CORE_CHECKPOINT_IDLE) { spl->checkpoint.pending_log = next; @@ -404,7 +417,8 @@ core_checkpoint_begin(core_handle *spl, bool32 force) spl->checkpoint.phase = CORE_CHECKPOINT_PENDING; spl->last_checkpoint_generation = spl->mt_ctxt.generation; next = NULL; // handed off to the checkpoint - armed = TRUE; + // Ours is the next completion to be counted. + ticket = spl->checkpoint.completions + 1; } platform_mutex_unlock(&spl->checkpoint_state_lock); @@ -413,7 +427,7 @@ core_checkpoint_begin(core_handle *spl, bool32 force) log_seal(next); log_dec_ref(spl->cc, &next_head); } - return armed; + return ticket; } /* The automatic, policy-driven arm, run after every rotation. */ @@ -564,7 +578,12 @@ core_maybe_complete_checkpoint(core_handle *spl) ZERO_CONTENTS(&spl->checkpoint.sealed_head); ZERO_CONTENTS(&spl->checkpoint.live_head); spl->checkpoint.cut_generation = 0; - spl->checkpoint.phase = CORE_CHECKPOINT_IDLE; + /* + * Count the completion only here, after log_dec_ref() above: waiters take + * this as proof the retired log's space is back. + */ + spl->checkpoint.completions++; + spl->checkpoint.phase = CORE_CHECKPOINT_IDLE; } else { // Leave the sealed slot intact and retry on a later incorporation. spl->checkpoint.phase = CORE_CHECKPOINT_INCORPORATING; @@ -2699,21 +2718,23 @@ core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns) { uint64 target = memtable_generation(&spl->mt_ctxt); - bool32 armed = core_checkpoint_begin(spl, TRUE /* force */); + uint64 ticket = core_checkpoint_begin(spl, TRUE /* force */); uint64 wait = 100; timestamp deadline = platform_get_timestamp(); while (TRUE) { bool32 incorporated = memtable_generation_retired(&spl->mt_ctxt) + 1 > target; - core_checkpoint_phase phase = core_checkpoint_phase_get(spl); + core_checkpoint_status status = core_checkpoint_status_get(spl); /* * Done once the target is durable-able and, if we started a checkpoint, - * it has completed (which is what freed the retired log). We only wait - * on a checkpoint we armed ourselves, so an unrelated one cannot hold us - * up. + * that specific checkpoint has completed -- which is what freed its + * retired log. Comparing against our own ticket rather than "nothing in + * flight" means neither an unrelated checkpoint nor one armed after ours + * can hold us up. */ - if (incorporated && (!armed || phase == CORE_CHECKPOINT_IDLE)) { + bool32 ours_completed = (ticket == 0) || (status.completions >= ticket); + if (incorporated && ours_completed) { break; } @@ -2722,8 +2743,9 @@ core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns) * still the active generation, or our checkpoint has yet to cut. Once * neither holds, the remaining work is just draining flushes. */ - bool32 needs_rotation = memtable_generation(&spl->mt_ctxt) == target - || (armed && phase == CORE_CHECKPOINT_PENDING); + bool32 needs_rotation = + memtable_generation(&spl->mt_ctxt) == target + || (ticket != 0 && status.phase == CORE_CHECKPOINT_PENDING); if (needs_rotation && rotation_timeout_ns <= platform_timestamp_elapsed(deadline)) { diff --git a/src/core.h b/src/core.h index 68942ad3..5da4f4b8 100644 --- a/src/core.h +++ b/src/core.h @@ -138,6 +138,16 @@ typedef struct core_checkpoint_state { // superblock already holds it and carries it into the sealed slot. uint64 live_start_generation; uint64 cut_generation; // complete once retired >= this + /* + * Checkpoints completed so far. Bumped only after the completion has freed + * the retired log, so it is the one observable meaning "that checkpoint's + * space is back" -- every superblock-visible signal is necessarily written + * before the free, since the superblock must stop naming a log before its + * extents are released. core_checkpoint_begin() hands out `completions + 1` + * as a ticket so a caller can wait for its own checkpoint rather than merely + * for "none in flight." + */ + uint64 completions; } core_checkpoint_state; typedef struct core_memtable_args { From 6c6c92254072892e87d9c7a1ddf75b90bbdbdca4 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Wed, 29 Jul 2026 13:53:30 -0700 Subject: [PATCH 53/64] Comment Signed-off-by: Rob Johnson --- src/core.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/core.c b/src/core.c index 6fea2845..1d628ae0 100644 --- a/src/core.c +++ b/src/core.c @@ -2742,6 +2742,12 @@ core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns) * Force a rotation if either half is still waiting on one: the target is * still the active generation, or our checkpoint has yet to cut. Once * neither holds, the remaining work is just draining flushes. + * + * The second clause is not redundant. Another thread can finalize the + * target in the window between reading it and arming, which leaves the + * first clause false while our checkpoint still needs a rotation to cut. + * On an otherwise idle system nothing would ever provide one, and the wait + * for our completion would never finish. */ bool32 needs_rotation = memtable_generation(&spl->mt_ctxt) == target From d2b03574e5ffb04f4aaced4abf78b5acdeaf10c9 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Wed, 29 Jul 2026 13:54:29 -0700 Subject: [PATCH 54/64] Comment Signed-off-by: Rob Johnson --- src/core.c | 50 ++++++++++++++++++++++++-------------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/src/core.c b/src/core.c index 1d628ae0..c4752405 100644 --- a/src/core.c +++ b/src/core.c @@ -206,10 +206,10 @@ core_log_to_superblock_log_head(log_head info, uint64 start_generation) * released; republishing an unchanged root is just the degenerate case of that, * so it needs no special handling. * - * Note this always publishes, even when the root is unchanged: callers stage log - * transitions into the image beforehand, and the generation bound can advance on - * its own (an empty generation retires without changing the root), so the root - * address alone is not a "nothing to do" test. + * Note this always publishes, even when the root is unchanged: callers stage + * log transitions into the image beforehand, and the generation bound can + * advance on its own (an empty generation retires without changing the root), + * so the root address alone is not a "nothing to do" test. */ static platform_status core_checkpoint_commit_current_root(core_handle *spl) @@ -270,14 +270,14 @@ core_checkpoint_commit_current_root(core_handle *spl) } /* - * The captured reference becomes the record's durable one, and the previously - * published root's reference is released. This is uniform even when the root - * did not change: that root's count is momentarily 2 (the record's plus ours) - * and the release brings it back to the record's single reference, so a - * same-root republish needs no special case and cannot grow the count. The - * publish barrier already committed the new root to the newer superblock slot, - * so the old slot is no longer the mount choice and releasing it cannot - * strand a torn-write fallback. + * The captured reference becomes the record's durable one, and the + * previously published root's reference is released. This is uniform even + * when the root did not change: that root's count is momentarily 2 (the + * record's plus ours) and the release brings it back to the record's single + * reference, so a same-root republish needs no special case and cannot grow + * the count. The publish barrier already committed the new root to the + * newer superblock slot, so the old slot is no longer the mount choice and + * releasing it cannot strand a torn-write fallback. */ snapshot.root_addr = 0; // transferred to the durable record if (old_root_addr != 0) { @@ -380,10 +380,10 @@ core_checkpoint_status_get(core_handle *spl) * Returns a completion ticket for the checkpoint this call armed: it has * finished, and freed its retired log, once checkpoint.completions reaches the * ticket. The ticket is captured under the same lock acquisition that arms, so - * it cannot miss or over-count a completion. Returns 0 if this call did not arm - * one, which is not an error -- only one checkpoint can be in flight at a time, - * and declining is the normal outcome when one already is. Tickets are 1-based, - * so 0 is unambiguous. + * it cannot miss or over-count a completion. Returns 0 if this call did not + * arm one, which is not an error -- only one checkpoint can be in flight at a + * time, and declining is the normal outcome when one already is. Tickets are + * 1-based, so 0 is unambiguous. */ static uint64 core_checkpoint_begin(core_handle *spl, bool32 force) @@ -2309,9 +2309,8 @@ core_mkfs(core_handle *spl, platform_status rc = core_locks_init(spl); if (!SUCCESS(rc)) { - platform_error_log( - "core_mkfs: lock initialization failed: %s\n", - platform_status_to_string(rc)); + platform_error_log("core_mkfs: lock initialization failed: %s\n", + platform_status_to_string(rc)); return rc; } @@ -2439,9 +2438,8 @@ core_mount(core_handle *spl, platform_status rc = core_locks_init(spl); if (!SUCCESS(rc)) { - platform_error_log( - "core_mount: lock initialization failed: %s\n", - platform_status_to_string(rc)); + platform_error_log("core_mount: lock initialization failed: %s\n", + platform_status_to_string(rc)); return rc; } @@ -2744,10 +2742,10 @@ core_checkpoint(core_handle *spl, uint64 rotation_timeout_ns) * neither holds, the remaining work is just draining flushes. * * The second clause is not redundant. Another thread can finalize the - * target in the window between reading it and arming, which leaves the - * first clause false while our checkpoint still needs a rotation to cut. - * On an otherwise idle system nothing would ever provide one, and the wait - * for our completion would never finish. + * target in the window between reading target and arming, which leaves + * the first clause false while our checkpoint still needs a rotation to + * cut. On an otherwise idle system nothing would ever provide one, and + * the wait for our completion would never finish. */ bool32 needs_rotation = memtable_generation(&spl->mt_ctxt) == target From edff0c9f508ba3c67aed822622a5bc999fba8460 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Wed, 29 Jul 2026 16:08:10 -0700 Subject: [PATCH 55/64] checkpoint policy cleanup Signed-off-by: Rob Johnson --- src/cache.h | 8 +++ src/clockcache.c | 8 +++ src/core.c | 103 +++++++++++++++++++++++++++++++++---- src/core.h | 43 ++++++++++++---- src/log.h | 17 ++++++ src/shard_log.c | 26 ++++++++++ src/shard_log.h | 7 +++ tests/unit/splinter_test.c | 93 ++++++++++++++++++++++++++++++--- 8 files changed, 276 insertions(+), 29 deletions(-) diff --git a/src/cache.h b/src/cache.h index 598df0f5..eb3b8813 100644 --- a/src/cache.h +++ b/src/cache.h @@ -62,6 +62,7 @@ typedef uint64 (*cache_config_generic_uint64_fn)(const cache_config *cfg); typedef struct cache_config_ops { cache_config_generic_uint64_fn page_size; cache_config_generic_uint64_fn extent_size; + cache_config_generic_uint64_fn capacity; } cache_config_ops; typedef struct cache_config { @@ -80,6 +81,13 @@ cache_config_extent_size(const cache_config *cfg) return cfg->ops->extent_size(cfg); } +/* Total bytes of pages the cache can hold. */ +static inline uint64 +cache_config_capacity(const cache_config *cfg) +{ + return cfg->ops->capacity(cfg); +} + static inline uint64 cache_config_pages_per_extent(const cache_config *cfg) { diff --git a/src/clockcache.c b/src/clockcache.c index bd59647b..b898fd95 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -3192,9 +3192,17 @@ clockcache_config_extent_size_virtual(const cache_config *cfg) return clockcache_config_extent_size(ccfg); } +uint64 +clockcache_config_capacity_virtual(const cache_config *cfg) +{ + clockcache_config *ccfg = (clockcache_config *)cfg; + return ccfg->capacity; +} + cache_config_ops clockcache_config_ops = { .page_size = clockcache_config_page_size_virtual, .extent_size = clockcache_config_extent_size_virtual, + .capacity = clockcache_config_capacity_virtual, }; page_handle * diff --git a/src/core.c b/src/core.c index c4752405..c71bd35f 100644 --- a/src/core.c +++ b/src/core.c @@ -72,7 +72,6 @@ core_locks_init(core_handle *spl) } ZERO_CONTENTS(&spl->checkpoint); // phase == CORE_CHECKPOINT_IDLE - spl->last_checkpoint_generation = 0; return STATUS_OK; } @@ -335,16 +334,26 @@ core_checkpoint_commit_current_root(core_handle *spl) *----------------------------------------------------------------------------- */ -/* Policy: should the next memtable rotation start a checkpoint? */ +/* + * Policy: should the next memtable rotation start a checkpoint? Triggered by the + * live log's size, since that is what a checkpoint reclaims -- and it is the only + * measure that tracks a workload which overwrites in place, filling the log + * without ever filling a memtable. + * + * Callers hold checkpoint_state_lock, which is also held while spl->log is + * swapped, so the log read below cannot race the cut. + */ static bool32 core_should_take_checkpoint(core_handle *spl) { - if (spl->cfg.checkpoint_generation_interval == 0) { + if (!spl->cfg.use_log || spl->cfg.checkpoint_log_size_bytes == 0) { return FALSE; } - uint64 current = spl->mt_ctxt.generation; - return current - spl->last_checkpoint_generation - >= spl->cfg.checkpoint_generation_interval; + /* + * log_get_size() reports bytes *appended*, so the fresh log a cut installs + * reads 0 and this settles rather than rotating in a loop. + */ + return log_get_size(spl->log) >= spl->cfg.checkpoint_log_size_bytes; } /* @@ -412,11 +421,10 @@ core_checkpoint_begin(core_handle *spl, bool32 force) uint64 ticket = 0; platform_mutex_lock(&spl->checkpoint_state_lock); if (spl->checkpoint.phase == CORE_CHECKPOINT_IDLE) { - spl->checkpoint.pending_log = next; - spl->checkpoint.live_head = next_head; - spl->checkpoint.phase = CORE_CHECKPOINT_PENDING; - spl->last_checkpoint_generation = spl->mt_ctxt.generation; - next = NULL; // handed off to the checkpoint + spl->checkpoint.pending_log = next; + spl->checkpoint.live_head = next_head; + spl->checkpoint.phase = CORE_CHECKPOINT_PENDING; + next = NULL; // handed off to the checkpoint // Ours is the next completion to be counted. ticket = spl->checkpoint.completions + 1; } @@ -437,6 +445,40 @@ core_checkpoint_maybe_begin(core_handle *spl) core_checkpoint_begin(spl, FALSE /* force */); } +/* + * Act on the size policy from the insert path. Called by core_insert() once the + * insert lock is released. + * + * A rotation is the only point at which the log can be cut, and a workload that + * overwrites in place updates the memtable without growing it -- so it may never + * fill a memtable, never rotate, and never give core_checkpoint_maybe_begin() + * (which only runs after a rotation) a chance to arm anything. Left to itself + * the log would grow without bound. + * + * Arming and then forcing the rotation cuts the log in a single rotation, since + * the rotate hook finds the checkpoint already PENDING. Only the thread that + * actually armed goes on to force, so concurrent inserters do not pile on. + * + * Not forcing the arm below is what makes this safe against a stale flag. The + * flag is only a hint -- sampled on some earlier insert, and readable by several + * threads at once -- so a thread can arrive here long after the log it observed + * was already cut. Passing force = FALSE has core_checkpoint_begin() re-check + * the policy under the state lock against the *current* log, which declines in + * exactly those cases (the fresh log reports zero bytes, or a checkpoint is still + * in flight) and proceeds only when another cut is genuinely due. Forcing here + * would instead cut again on a log that no longer needs it. + */ +static void +core_maybe_cut_oversized_log(core_handle *spl) +{ + if (!spl->log_reached_threshold) { + return; + } + if (core_checkpoint_begin(spl, FALSE /* force */) != 0) { + memtable_force_rotation(&spl->mt_ctxt); + } +} + /* * Begin, step 2 (inside the rotation critical section, insert lock held * exclusively): swap the pre-created live log in. Registered as @@ -465,6 +507,15 @@ core_rotate_log(void *arg, uint64 finalized_generation) spl->checkpoint.live_start_generation = finalized_generation + 1; spl->checkpoint.cut_generation = finalized_generation; spl->checkpoint.phase = CORE_CHECKPOINT_SEALING; + + /* + * The size hint described the log we just retired; the fresh one has had + * nothing appended. This is the only place the hint is cleared, and the + * insert lock is held exclusively here, so it cannot race the stores in + * core_log_insert(). A rotation that does not cut leaves the hint alone, + * which is correct: the same log is still live and still oversized. + */ + spl->log_reached_threshold = FALSE; } platform_mutex_unlock(&spl->checkpoint_state_lock); } @@ -823,6 +874,26 @@ core_log_insert(core_handle *spl, log_msg, memtable_generation, insert_results->leaf_generation); + + /* + * Sample the size policy while we still hold the shared insert lock, which is + * what makes reading spl->log safe (the live log is only swapped from inside + * the rotation critical section, which holds that lock exclusively). + * core_insert() acts on this once the lock is released. + * + * Only ever set it here, never clear it: this runs on every logged insert on + * every thread, and writing a shared field that often would bounce its cache + * line between cores for no reason. Leaving the common case read-only keeps + * the line shared. core_rotate_log() clears the flag when it cuts the log, + * which it does under the insert lock held exclusively -- so the clear cannot + * race this store. + */ + if (spl->cfg.checkpoint_log_size_bytes != 0 + && log_get_size(spl->log) >= spl->cfg.checkpoint_log_size_bytes) + { + spl->log_reached_threshold = TRUE; + } + return log_rc == 0 ? STATUS_OK : (platform_status){.r = log_rc}; } @@ -2008,6 +2079,8 @@ core_insert(core_handle *spl, btree_insert_results_deinit(&insert_results); task_perform_one_if_needed(spl->ts, spl->cfg.queue_scale_percent); + // The insert lock is released by here, so this may force a rotation. + core_maybe_cut_oversized_log(spl); if (spl->cfg.use_stats) { switch (message_class(data)) { @@ -3270,6 +3343,14 @@ core_config_init(core_config *core_cfg, core_cfg->verbose_logging_enabled = verbose_logging; core_cfg->log_handle = log_handle; + /* + * Rotate the log once it is as large as the cache. Replaying a log that far + * exceeds the cache gains nothing (the pages cannot stay resident), and this + * bounds both recovery time and the log's footprint. Callers that manage + * checkpoints themselves set this to 0. + */ + core_cfg->checkpoint_log_size_bytes = cache_config_capacity(cache_cfg); + memtable_config_init(&core_cfg->mt_cfg, core_cfg->btree_cfg, CORE_NUM_MEMTABLES, diff --git a/src/core.h b/src/core.h index 5da4f4b8..91f4d365 100644 --- a/src/core.h +++ b/src/core.h @@ -51,10 +51,20 @@ typedef struct core_config { data_config *data_cfg; bool32 use_log; log_config *log_cfg; - // Automatic-checkpoint policy: take a checkpoint (rotate the log and advance - // the durable root) once this many memtable generations have been finalized - // since the last one. 0 disables automatic checkpoints. - uint64 checkpoint_generation_interval; + /* + * Automatic-checkpoint policy: take a checkpoint (rotate the log and advance + * the durable root) once the live log reaches this many bytes. 0 disables + * automatic checkpoints, leaving durability and log reclamation entirely to + * explicit core_checkpoint() calls. Defaults to the cache size. + * + * Sizing the trigger by log bytes rather than by memtable generations matters + * because the two are independent: a workload that repeatedly overwrites the + * same keys updates the memtable in place, so it may never fill a memtable or + * advance a generation, while every write still appends to the log. A + * generation-based trigger would never fire and the log would grow without + * bound. + */ + uint64 checkpoint_log_size_bytes; trunk_config *trunk_node_cfg; // verbose logging @@ -189,14 +199,27 @@ struct core_handle { /* * Incorporation-driven checkpoint state. checkpoint_state_lock guards the - * fields of `checkpoint` and last_checkpoint_generation (and is held while - * `log` is swapped); it is only ever held for brief, I/O-free updates (never - * across a barrier), so taking it inside the memtable rotation critical - * section cannot stall inserts on I/O. + * fields of `checkpoint` (and is held while `log` is swapped); it is only ever + * held for brief, I/O-free updates (never across a barrier), so taking it + * inside the memtable rotation critical section cannot stall inserts on I/O. */ platform_mutex checkpoint_state_lock; core_checkpoint_state checkpoint; - uint64 last_checkpoint_generation; + + /* + * Hint that the live log has reached checkpoint_log_size_bytes. Set by + * core_log_insert() -- which already holds the shared insert lock, the same + * lock that excludes the log swap, so it can read `log` safely -- and acted on + * by core_insert() once that lock is released. Cleared only by + * core_rotate_log() when it cuts the log, under the insert lock held + * exclusively, so set and clear cannot race. + * + * Set-only on the insert path so the common case does no store and the cache + * line stays shared across cores. Being merely a hint, a stale TRUE costs + * nothing: core_maybe_cut_oversized_log() re-checks the policy against the + * current log before acting. + */ + bool32 log_reached_threshold; core_stats *stats; @@ -323,7 +346,7 @@ core_mount(core_handle *spl, * both are done. Safe to call on a running system with concurrent inserts. * * Reclamation makes this sufficient on its own, so an application can set - * checkpoint_generation_interval to 0 (no automatic checkpoints) and manage + * checkpoint_log_size_bytes to 0 (no automatic checkpoints) and manage * durability and log space entirely through this call. * * Both halves need a memtable rotation, which insert traffic normally triggers. diff --git a/src/log.h b/src/log.h index 8e712881..15263a7b 100644 --- a/src/log.h +++ b/src/log.h @@ -54,11 +54,21 @@ typedef platform_status (*log_seal_fn)(log_handle *log); * mid-stream can find the stream for replay. */ typedef log_head (*log_head_fn)(log_handle *log); +/* + * Bytes appended to the stream so far, so a caller can decide when to retire it. + * Excludes the implementation's fixed per-stream overhead: a stream that has had + * nothing written to it reports 0, which keeps a size-triggered policy from + * firing on a brand-new stream no matter how small its threshold. A + * conservative measure otherwise -- space is counted as it is reserved, so this + * rounds up to whatever allocation unit the implementation uses. + */ +typedef uint64 (*log_size_fn)(log_handle *log); typedef struct log_ops { log_write_fn write; log_seal_fn seal; log_head_fn head; + log_size_fn size; } log_ops; // to sub-class log, make a log_handle your first field @@ -96,6 +106,13 @@ log_get_head(log_handle *log) return log->ops->head(log); } +/* Bytes the stream currently occupies on disk. See log_size_fn. */ +static inline uint64 +log_get_size(log_handle *log) +{ + return log->ops->size(log); +} + /* * A log_handle is created by the concrete log implementation -- e.g. * shard_log_create() -- and then driven through the abstract ops above; it is diff --git a/src/shard_log.c b/src/shard_log.c index 58a518fe..2255990b 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -42,6 +42,12 @@ shard_log_pages_per_extent(shard_log_config *cfg) return cache_config_pages_per_extent(cfg->cache_cfg); } +static inline uint64 +shard_log_extent_size(shard_log_config *cfg) +{ + return cache_config_extent_size(cfg->cache_cfg); +} + static inline checksum128 shard_log_checksum(shard_log_config *cfg, page_handle *page) { @@ -391,10 +397,27 @@ shard_log_next_extent_addr(shard_log_config *cfg, page_handle *page) return hdr->next_extent_addr; } +/* + * Bytes appended to the stream so far. The mini-allocator already tracks the + * extents it has handed out across all of the stream's batches (data and blob), + * counting each as it is reserved -- the same measure memtable_is_full() uses for + * a memtable. Subtracting the fixed overhead recorded at init means a fresh + * stream reports 0, so a caller comparing against a threshold cannot be tricked + * into rotating a stream that has had nothing written to it. + */ +uint64 +shard_log_get_size(log_handle *logh) +{ + shard_log *log = (shard_log *)logh; + return (mini_num_extents(&log->mini) - log->initial_extents) + * shard_log_extent_size(log->cfg); +} + static log_ops shard_log_ops = { .write = shard_log_write, .seal = shard_log_seal, .head = shard_log_get_head, + .size = shard_log_get_size, }; static platform_status @@ -429,6 +452,9 @@ shard_log_init(shard_log *log, cache *cc, shard_log_config *cfg) // platform_default_log("addr: %lu meta_head: %lu\n", log->addr, // log->meta_head); + // Baseline for shard_log_get_size(): the stream's fixed overhead. + log->initial_extents = mini_num_extents(&log->mini); + return STATUS_OK; } diff --git a/src/shard_log.h b/src/shard_log.h index 9bb1a5ef..efc74f8e 100644 --- a/src/shard_log.h +++ b/src/shard_log.h @@ -47,6 +47,13 @@ typedef struct shard_log { uint64 addr; uint64 meta_head; uint64 magic; + /* + * Extents the mini-allocator held once the stream was initialized -- its + * fixed per-stream overhead (a metadata extent plus one per batch). + * shard_log_get_size() subtracts it so a fresh stream reports zero bytes + * appended. + */ + uint64 initial_extents; } shard_log; typedef struct log_entry log_entry; diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index ddef59f5..3bb3c8fb 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -331,7 +331,7 @@ CTEST2(splinter, test_self_managed_checkpoints_reclaim_log_space) allocator *alp = (allocator *)&data->al; data->system_cfg->splinter_cfg.use_log = TRUE; // No automatic checkpoints: this test drives them all itself. - data->system_cfg->splinter_cfg.checkpoint_generation_interval = 0; + data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = 0; core_handle spl; platform_status rc = core_mkfs(&spl, @@ -396,15 +396,15 @@ CTEST2(splinter, test_self_managed_checkpoints_reclaim_log_space) * double-freed. The caller sets up the task system (foreground or background). */ static void -run_auto_checkpoint_workload(void *datap, uint64 interval) +run_auto_checkpoint_workload(void *datap, uint64 log_size_threshold) { struct CTEST_IMPL_DATA_SNAME(splinter) *data = (struct CTEST_IMPL_DATA_SNAME(splinter) *)datap; allocator *alp = (allocator *)&data->al; data->system_cfg->splinter_cfg.use_log = TRUE; - // Rotate the log / advance the durable root every `interval` generations. - data->system_cfg->splinter_cfg.checkpoint_generation_interval = interval; + // Rotate the log / advance the durable root once the log reaches this size. + data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = log_size_threshold; core_handle spl; platform_status rc = core_mkfs(&spl, @@ -424,9 +424,10 @@ run_auto_checkpoint_workload(void *datap, uint64 interval) rc = task_perform_until_quiescent(spl.ts); ASSERT_TRUE(SUCCESS(rc)); - if (interval != 0) { - // The inserts must have driven at least one automatic checkpoint. - ASSERT_NOT_EQUAL(0, spl.last_checkpoint_generation); + if (log_size_threshold != 0) { + // The inserts must have driven at least one automatic checkpoint through + // to completion. + ASSERT_NOT_EQUAL(0, spl.checkpoint.completions); // At rest a checkpoint has either completed (IDLE) or been armed for a // rotation that idle never triggered (PENDING); both leave no sealed log. @@ -471,7 +472,83 @@ run_auto_checkpoint_workload(void *datap, uint64 interval) */ CTEST2(splinter, test_auto_checkpoint) { - run_auto_checkpoint_workload(data, 2); + // One extent: small enough that the test workload drives several rotations. + run_auto_checkpoint_workload(data, 2 * data->system_cfg->io_cfg.extent_size); +} + +/* + * The reason the policy is sized in log bytes rather than memtable generations. + * + * Repeatedly overwriting one key updates the memtable btree in place, so it never + * accumulates extents, never becomes "full", and never rotates -- the generation + * stays put for the whole workload. Every write still appends to the log, so the + * log grows without bound. A generation-based trigger could never fire here; the + * size-based one must. + */ +CTEST2(splinter, test_auto_checkpoint_on_overwrites) +{ + allocator *alp = (allocator *)&data->al; + data->system_cfg->splinter_cfg.use_log = TRUE; + data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = + 2 * data->system_cfg->io_cfg.extent_size; + + core_handle spl; + platform_status rc = core_mkfs(&spl, + &data->system_cfg->splinter_cfg, + alp, + (cache *)data->clock_cache, + data->io, + &data->tasks, + test_generate_allocator_root_id(), + data->hid); + ASSERT_TRUE(SUCCESS(rc)); + + uint64 start_generation = memtable_generation(&spl.mt_ctxt); + + // Hammer a single key. Enough writes to push the log well past one extent. + DECLARE_AUTO_KEY_BUFFER(keybuf, data->hid); + merge_accumulator msg; + merge_accumulator_init(&msg, data->hid); + const uint64 num_overwrites = 20000; + for (uint64 i = 0; i < num_overwrites; i++) { + test_key(&keybuf, TEST_RANDOM, 0, 0, 0, data->workload_cfg->key_size, 0); + generate_test_message(&data->gen, i, &msg); + rc = core_insert( + &spl, key_buffer_key(&keybuf), merge_accumulator_to_message(&msg), NULL); + ASSERT_TRUE(SUCCESS(rc)); + } + + rc = task_perform_until_quiescent(spl.ts); + ASSERT_TRUE(SUCCESS(rc)); + + /* + * The workload never filled a memtable on its own, so a generation-based + * policy would have had nothing to trigger on: any generation advance here + * came from a checkpoint forcing a rotation, not from the memtable filling. + */ + ASSERT_NOT_EQUAL(0, + spl.checkpoint.completions, + "overwrite-only workload did not trigger a checkpoint; " + "generation went %lu -> %lu\n", + start_generation, + memtable_generation(&spl.mt_ctxt)); + + // The surviving value must be the last one written. + lookup_result qdata; + lookup_result_init( + &qdata, spl.cfg.data_cfg, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + test_key(&keybuf, TEST_RANDOM, 0, 0, 0, data->workload_cfg->key_size, 0); + rc = core_lookup(&spl, key_buffer_key(&keybuf), &qdata); + ASSERT_TRUE(SUCCESS(rc)); + generate_test_message(&data->gen, num_overwrites - 1, &msg); + ASSERT_EQUAL(0, + message_lex_cmp( + merge_accumulator_to_message(&msg), + merge_accumulator_to_message(lookup_result_accumulator(&qdata)))); + lookup_result_deinit(&qdata); + merge_accumulator_deinit(&msg); + + core_destroy(&spl); } /* From 7839ee03ceca176113b6d01c665e232e004ed185 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Wed, 29 Jul 2026 16:16:12 -0700 Subject: [PATCH 56/64] checkpoint policy cleanup Signed-off-by: Rob Johnson --- include/splinterdb/splinterdb.h | 16 ++++++++++++++++ src/cache.h | 8 -------- src/clockcache.c | 8 -------- src/core.c | 18 ++++++------------ src/core.h | 1 + src/splinterdb.c | 9 +++++++++ tests/functional/test.h | 1 + 7 files changed, 33 insertions(+), 28 deletions(-) diff --git a/include/splinterdb/splinterdb.h b/include/splinterdb/splinterdb.h index abb36455..d9be5a08 100644 --- a/include/splinterdb/splinterdb.h +++ b/include/splinterdb/splinterdb.h @@ -101,6 +101,22 @@ typedef struct splinterdb_config { // log _Bool use_log; + // Automatic checkpoints: once the write-ahead log has grown by this many + // bytes, SplinterDB takes a checkpoint, which folds the logged updates into + // the durable tree and reclaims that log's space. This bounds both how much + // log a crash has to replay and how much space the log occupies. + // + // The trigger is sized in log bytes rather than in updates because the two + // are independent: a workload that repeatedly overwrites the same keys grows + // the log with every write while barely growing the tree. + // + // Zero selects a default of one cache's worth of log, on the reasoning that + // replaying much more log than the cache can hold has little to gain. To + // effectively disable automatic checkpoints -- for an application that calls + // for them itself, and accepts that the log grows until it does -- set this + // very large (UINT64_MAX). + uint64 checkpoint_log_size_bytes; + // splinter uint64 memtable_capacity; uint64 fanout; diff --git a/src/cache.h b/src/cache.h index eb3b8813..598df0f5 100644 --- a/src/cache.h +++ b/src/cache.h @@ -62,7 +62,6 @@ typedef uint64 (*cache_config_generic_uint64_fn)(const cache_config *cfg); typedef struct cache_config_ops { cache_config_generic_uint64_fn page_size; cache_config_generic_uint64_fn extent_size; - cache_config_generic_uint64_fn capacity; } cache_config_ops; typedef struct cache_config { @@ -81,13 +80,6 @@ cache_config_extent_size(const cache_config *cfg) return cfg->ops->extent_size(cfg); } -/* Total bytes of pages the cache can hold. */ -static inline uint64 -cache_config_capacity(const cache_config *cfg) -{ - return cfg->ops->capacity(cfg); -} - static inline uint64 cache_config_pages_per_extent(const cache_config *cfg) { diff --git a/src/clockcache.c b/src/clockcache.c index b898fd95..bd59647b 100644 --- a/src/clockcache.c +++ b/src/clockcache.c @@ -3192,17 +3192,9 @@ clockcache_config_extent_size_virtual(const cache_config *cfg) return clockcache_config_extent_size(ccfg); } -uint64 -clockcache_config_capacity_virtual(const cache_config *cfg) -{ - clockcache_config *ccfg = (clockcache_config *)cfg; - return ccfg->capacity; -} - cache_config_ops clockcache_config_ops = { .page_size = clockcache_config_page_size_virtual, .extent_size = clockcache_config_extent_size_virtual, - .capacity = clockcache_config_capacity_virtual, }; page_handle * diff --git a/src/core.c b/src/core.c index c71bd35f..db6cdd71 100644 --- a/src/core.c +++ b/src/core.c @@ -3322,6 +3322,7 @@ core_config_init(core_config *core_cfg, uint64 queue_scale_percent, uint64 prefetch_budget, bool32 use_log, + uint64 checkpoint_log_size_bytes, bool32 use_stats, bool32 verbose_logging, platform_log_handle *log_handle) @@ -3338,18 +3339,11 @@ core_config_init(core_config *core_cfg, core_cfg->queue_scale_percent = queue_scale_percent; core_cfg->prefetch_budget = prefetch_budget; - core_cfg->use_log = use_log; - core_cfg->use_stats = use_stats; - core_cfg->verbose_logging_enabled = verbose_logging; - core_cfg->log_handle = log_handle; - - /* - * Rotate the log once it is as large as the cache. Replaying a log that far - * exceeds the cache gains nothing (the pages cannot stay resident), and this - * bounds both recovery time and the log's footprint. Callers that manage - * checkpoints themselves set this to 0. - */ - core_cfg->checkpoint_log_size_bytes = cache_config_capacity(cache_cfg); + core_cfg->use_log = use_log; + core_cfg->checkpoint_log_size_bytes = checkpoint_log_size_bytes; + core_cfg->use_stats = use_stats; + core_cfg->verbose_logging_enabled = verbose_logging; + core_cfg->log_handle = log_handle; memtable_config_init(&core_cfg->mt_cfg, core_cfg->btree_cfg, diff --git a/src/core.h b/src/core.h index 91f4d365..ff1e9bc7 100644 --- a/src/core.h +++ b/src/core.h @@ -421,6 +421,7 @@ core_config_init(core_config *trunk_cfg, uint64 queue_scale_percent, uint64 prefetch_budget, bool32 use_log, + uint64 checkpoint_log_size_bytes, bool32 use_stats, bool32 verbose_logging, platform_log_handle *log_handle); diff --git a/src/splinterdb.c b/src/splinterdb.c index 35ce2384..653f4bd4 100644 --- a/src/splinterdb.c +++ b/src/splinterdb.c @@ -164,6 +164,14 @@ splinterdb_config_set_defaults(splinterdb_config *cfg) if (!cfg->prefetch_budget) { cfg->prefetch_budget = CORE_DEFAULT_PREFETCH_BUDGET; } + if (!cfg->checkpoint_log_size_bytes) { + /* + * Checkpoint once the log has grown by a cache's worth: replaying much + * more log than the cache can hold gains little, since those pages cannot + * stay resident anyway. + */ + cfg->checkpoint_log_size_bytes = cfg->cache_size; + } } static platform_status @@ -311,6 +319,7 @@ splinterdb_init_config(const splinterdb_config *kvs_cfg, // IN cfg.queue_scale_percent, cfg.prefetch_budget, cfg.use_log, + cfg.checkpoint_log_size_bytes, cfg.use_stats, FALSE, Platform_default_log_handle); diff --git a/tests/functional/test.h b/tests/functional/test.h index 82ab07ce..d107a24c 100644 --- a/tests/functional/test.h +++ b/tests/functional/test.h @@ -308,6 +308,7 @@ test_config_init(system_config *system_cfg, // OUT master_cfg->queue_scale_percent, master_cfg->prefetch_budget, master_cfg->use_log, + master_cfg->cache_capacity, master_cfg->use_stats, master_cfg->verbose_logging_enabled, master_cfg->log_handle); From 9718bb542b5d34d1b1ad6021ac8b8acc7f59a0f4 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Wed, 29 Jul 2026 16:16:24 -0700 Subject: [PATCH 57/64] formatting Signed-off-by: Rob Johnson --- src/core.c | 49 +++++++++++++++++++------------------- src/core.h | 23 +++++++++--------- src/log.h | 8 +++---- src/shard_log.c | 8 +++---- tests/unit/splinter_test.c | 25 ++++++++++--------- 5 files changed, 59 insertions(+), 54 deletions(-) diff --git a/src/core.c b/src/core.c index db6cdd71..8d31f593 100644 --- a/src/core.c +++ b/src/core.c @@ -335,10 +335,10 @@ core_checkpoint_commit_current_root(core_handle *spl) */ /* - * Policy: should the next memtable rotation start a checkpoint? Triggered by the - * live log's size, since that is what a checkpoint reclaims -- and it is the only - * measure that tracks a workload which overwrites in place, filling the log - * without ever filling a memtable. + * Policy: should the next memtable rotation start a checkpoint? Triggered by + * the live log's size, since that is what a checkpoint reclaims -- and it is + * the only measure that tracks a workload which overwrites in place, filling + * the log without ever filling a memtable. * * Callers hold checkpoint_state_lock, which is also held while spl->log is * swapped, so the log read below cannot race the cut. @@ -446,27 +446,28 @@ core_checkpoint_maybe_begin(core_handle *spl) } /* - * Act on the size policy from the insert path. Called by core_insert() once the - * insert lock is released. + * Act on the size policy from the insert path. Called by core_insert() once + * the insert lock is released. * * A rotation is the only point at which the log can be cut, and a workload that - * overwrites in place updates the memtable without growing it -- so it may never - * fill a memtable, never rotate, and never give core_checkpoint_maybe_begin() - * (which only runs after a rotation) a chance to arm anything. Left to itself - * the log would grow without bound. + * overwrites in place updates the memtable without growing it -- so it may + * never fill a memtable, never rotate, and never give + * core_checkpoint_maybe_begin() (which only runs after a rotation) a chance to + * arm anything. Left to itself the log would grow without bound. * * Arming and then forcing the rotation cuts the log in a single rotation, since * the rotate hook finds the checkpoint already PENDING. Only the thread that * actually armed goes on to force, so concurrent inserters do not pile on. * * Not forcing the arm below is what makes this safe against a stale flag. The - * flag is only a hint -- sampled on some earlier insert, and readable by several - * threads at once -- so a thread can arrive here long after the log it observed - * was already cut. Passing force = FALSE has core_checkpoint_begin() re-check - * the policy under the state lock against the *current* log, which declines in - * exactly those cases (the fresh log reports zero bytes, or a checkpoint is still - * in flight) and proceeds only when another cut is genuinely due. Forcing here - * would instead cut again on a log that no longer needs it. + * flag is only a hint -- sampled on some earlier insert, and readable by + * several threads at once -- so a thread can arrive here long after the log it + * observed was already cut. Passing force = FALSE has core_checkpoint_begin() + * re-check the policy under the state lock against the *current* log, which + * declines in exactly those cases (the fresh log reports zero bytes, or a + * checkpoint is still in flight) and proceeds only when another cut is + * genuinely due. Forcing here would instead cut again on a log that no longer + * needs it. */ static void core_maybe_cut_oversized_log(core_handle *spl) @@ -876,17 +877,17 @@ core_log_insert(core_handle *spl, insert_results->leaf_generation); /* - * Sample the size policy while we still hold the shared insert lock, which is - * what makes reading spl->log safe (the live log is only swapped from inside - * the rotation critical section, which holds that lock exclusively). + * Sample the size policy while we still hold the shared insert lock, which + * is what makes reading spl->log safe (the live log is only swapped from + * inside the rotation critical section, which holds that lock exclusively). * core_insert() acts on this once the lock is released. * * Only ever set it here, never clear it: this runs on every logged insert on * every thread, and writing a shared field that often would bounce its cache * line between cores for no reason. Leaving the common case read-only keeps * the line shared. core_rotate_log() clears the flag when it cuts the log, - * which it does under the insert lock held exclusively -- so the clear cannot - * race this store. + * which it does under the insert lock held exclusively -- so the clear + * cannot race this store. */ if (spl->cfg.checkpoint_log_size_bytes != 0 && log_get_size(spl->log) >= spl->cfg.checkpoint_log_size_bytes) @@ -3337,8 +3338,8 @@ core_config_init(core_config *core_cfg, core_cfg->trunk_node_cfg = trunk_node_cfg; core_cfg->log_cfg = log_cfg; - core_cfg->queue_scale_percent = queue_scale_percent; - core_cfg->prefetch_budget = prefetch_budget; + core_cfg->queue_scale_percent = queue_scale_percent; + core_cfg->prefetch_budget = prefetch_budget; core_cfg->use_log = use_log; core_cfg->checkpoint_log_size_bytes = checkpoint_log_size_bytes; core_cfg->use_stats = use_stats; diff --git a/src/core.h b/src/core.h index ff1e9bc7..3ac52454 100644 --- a/src/core.h +++ b/src/core.h @@ -57,12 +57,12 @@ typedef struct core_config { * automatic checkpoints, leaving durability and log reclamation entirely to * explicit core_checkpoint() calls. Defaults to the cache size. * - * Sizing the trigger by log bytes rather than by memtable generations matters - * because the two are independent: a workload that repeatedly overwrites the - * same keys updates the memtable in place, so it may never fill a memtable or - * advance a generation, while every write still appends to the log. A - * generation-based trigger would never fire and the log would grow without - * bound. + * Sizing the trigger by log bytes rather than by memtable generations + * matters because the two are independent: a workload that repeatedly + * overwrites the same keys updates the memtable in place, so it may never + * fill a memtable or advance a generation, while every write still appends + * to the log. A generation-based trigger would never fire and the log would + * grow without bound. */ uint64 checkpoint_log_size_bytes; trunk_config *trunk_node_cfg; @@ -199,9 +199,10 @@ struct core_handle { /* * Incorporation-driven checkpoint state. checkpoint_state_lock guards the - * fields of `checkpoint` (and is held while `log` is swapped); it is only ever - * held for brief, I/O-free updates (never across a barrier), so taking it - * inside the memtable rotation critical section cannot stall inserts on I/O. + * fields of `checkpoint` (and is held while `log` is swapped); it is only + * ever held for brief, I/O-free updates (never across a barrier), so taking + * it inside the memtable rotation critical section cannot stall inserts on + * I/O. */ platform_mutex checkpoint_state_lock; core_checkpoint_state checkpoint; @@ -209,8 +210,8 @@ struct core_handle { /* * Hint that the live log has reached checkpoint_log_size_bytes. Set by * core_log_insert() -- which already holds the shared insert lock, the same - * lock that excludes the log swap, so it can read `log` safely -- and acted on - * by core_insert() once that lock is released. Cleared only by + * lock that excludes the log swap, so it can read `log` safely -- and acted + * on by core_insert() once that lock is released. Cleared only by * core_rotate_log() when it cuts the log, under the insert lock held * exclusively, so set and clear cannot race. * diff --git a/src/log.h b/src/log.h index 15263a7b..09ce8afe 100644 --- a/src/log.h +++ b/src/log.h @@ -55,10 +55,10 @@ typedef platform_status (*log_seal_fn)(log_handle *log); */ typedef log_head (*log_head_fn)(log_handle *log); /* - * Bytes appended to the stream so far, so a caller can decide when to retire it. - * Excludes the implementation's fixed per-stream overhead: a stream that has had - * nothing written to it reports 0, which keeps a size-triggered policy from - * firing on a brand-new stream no matter how small its threshold. A + * Bytes appended to the stream so far, so a caller can decide when to retire + * it. Excludes the implementation's fixed per-stream overhead: a stream that + * has had nothing written to it reports 0, which keeps a size-triggered policy + * from firing on a brand-new stream no matter how small its threshold. A * conservative measure otherwise -- space is counted as it is reserved, so this * rounds up to whatever allocation unit the implementation uses. */ diff --git a/src/shard_log.c b/src/shard_log.c index 2255990b..3f6934d9 100644 --- a/src/shard_log.c +++ b/src/shard_log.c @@ -400,10 +400,10 @@ shard_log_next_extent_addr(shard_log_config *cfg, page_handle *page) /* * Bytes appended to the stream so far. The mini-allocator already tracks the * extents it has handed out across all of the stream's batches (data and blob), - * counting each as it is reserved -- the same measure memtable_is_full() uses for - * a memtable. Subtracting the fixed overhead recorded at init means a fresh - * stream reports 0, so a caller comparing against a threshold cannot be tricked - * into rotating a stream that has had nothing written to it. + * counting each as it is reserved -- the same measure memtable_is_full() uses + * for a memtable. Subtracting the fixed overhead recorded at init means a + * fresh stream reports 0, so a caller comparing against a threshold cannot be + * tricked into rotating a stream that has had nothing written to it. */ uint64 shard_log_get_size(log_handle *logh) diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index 3bb3c8fb..0f368cdc 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -404,7 +404,8 @@ run_auto_checkpoint_workload(void *datap, uint64 log_size_threshold) allocator *alp = (allocator *)&data->al; data->system_cfg->splinter_cfg.use_log = TRUE; // Rotate the log / advance the durable root once the log reaches this size. - data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = log_size_threshold; + data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = + log_size_threshold; core_handle spl; platform_status rc = core_mkfs(&spl, @@ -479,11 +480,11 @@ CTEST2(splinter, test_auto_checkpoint) /* * The reason the policy is sized in log bytes rather than memtable generations. * - * Repeatedly overwriting one key updates the memtable btree in place, so it never - * accumulates extents, never becomes "full", and never rotates -- the generation - * stays put for the whole workload. Every write still appends to the log, so the - * log grows without bound. A generation-based trigger could never fire here; the - * size-based one must. + * Repeatedly overwriting one key updates the memtable btree in place, so it + * never accumulates extents, never becomes "full", and never rotates -- the + * generation stays put for the whole workload. Every write still appends to + * the log, so the log grows without bound. A generation-based trigger could + * never fire here; the size-based one must. */ CTEST2(splinter, test_auto_checkpoint_on_overwrites) { @@ -513,8 +514,10 @@ CTEST2(splinter, test_auto_checkpoint_on_overwrites) for (uint64 i = 0; i < num_overwrites; i++) { test_key(&keybuf, TEST_RANDOM, 0, 0, 0, data->workload_cfg->key_size, 0); generate_test_message(&data->gen, i, &msg); - rc = core_insert( - &spl, key_buffer_key(&keybuf), merge_accumulator_to_message(&msg), NULL); + rc = core_insert(&spl, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); ASSERT_TRUE(SUCCESS(rc)); } @@ -542,9 +545,9 @@ CTEST2(splinter, test_auto_checkpoint_on_overwrites) ASSERT_TRUE(SUCCESS(rc)); generate_test_message(&data->gen, num_overwrites - 1, &msg); ASSERT_EQUAL(0, - message_lex_cmp( - merge_accumulator_to_message(&msg), - merge_accumulator_to_message(lookup_result_accumulator(&qdata)))); + message_lex_cmp(merge_accumulator_to_message(&msg), + merge_accumulator_to_message( + lookup_result_accumulator(&qdata)))); lookup_result_deinit(&qdata); merge_accumulator_deinit(&msg); From bc8981d6a44bb2b234f96a5dd286fd7329e779a8 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Thu, 30 Jul 2026 11:19:57 -0700 Subject: [PATCH 58/64] add stats and config Signed-off-by: Rob Johnson --- src/core.c | 8 ++++++++ src/core.h | 9 +++++++++ tests/config.c | 12 ++++++++++++ tests/config.h | 8 ++++++++ tests/functional/test.h | 7 ++++++- tests/unit/splinter_test.c | 18 ++++++++++++++++++ 6 files changed, 61 insertions(+), 1 deletion(-) diff --git a/src/core.c b/src/core.c index 8d31f593..84134b93 100644 --- a/src/core.c +++ b/src/core.c @@ -641,6 +641,11 @@ core_maybe_complete_checkpoint(core_handle *spl) spl->checkpoint.phase = CORE_CHECKPOINT_INCORPORATING; } platform_mutex_unlock(&spl->checkpoint_state_lock); + + // Outside the lock: this is a per-thread counter, so it needs none. + if (SUCCESS(rc) && spl->cfg.use_stats) { + spl->stats[platform_get_tid()].checkpoints_completed++; + } return rc; } @@ -3125,6 +3130,7 @@ core_print_insertion_stats(platform_log_handle *log_handle, const core_handle *s spl->stats[thr_i].memtable_flush_time_max_ns; } global->memtable_flush_root_full += spl->stats[thr_i].memtable_flush_root_full; + global->checkpoints_completed += spl->stats[thr_i].checkpoints_completed; } platform_log(log_handle, "Overall Statistics\n"); @@ -3136,6 +3142,8 @@ core_print_insertion_stats(platform_log_handle *log_handle, const core_handle *s platform_log(log_handle, "------------------------------------------------------------------------------------\n"); platform_log(log_handle, "| root stalls: %10lu\n", global->memtable_flush_root_full); platform_log(log_handle, "------------------------------------------------------------------------------------\n"); + platform_log(log_handle, "| checkpoints: %10lu\n", global->checkpoints_completed); + platform_log(log_handle, "------------------------------------------------------------------------------------\n"); platform_log(log_handle, "\n"); platform_log(log_handle, "Latency Histogram Statistics\n"); diff --git a/src/core.h b/src/core.h index 3ac52454..27713f47 100644 --- a/src/core.h +++ b/src/core.h @@ -97,6 +97,15 @@ typedef struct core_stats { uint64 discarded_deletes; + /* + * Checkpoints that ran to completion -- meaning the durable root advanced and + * the retired log's space came back. Counted here for reporting only; the + * authoritative count the checkpoint machinery waits on is + * core_checkpoint_state.completions, which must stay monotonic and so is not + * affected by core_reset_stats(). + */ + uint64 checkpoints_completed; + uint64 lookups_found; uint64 lookups_not_found; } PLATFORM_CACHELINE_ALIGNED core_stats; diff --git a/tests/config.c b/tests/config.c index 1f2f873d..d9463cda 100644 --- a/tests/config.c +++ b/tests/config.c @@ -83,6 +83,8 @@ config_set_defaults(master_config *cfg) .filter_hash_size = TEST_CONFIG_DEFAULT_FILTER_HASH_SIZE, .filter_log_index_size = TEST_CONFIG_DEFAULT_FILTER_LOG_INDEX_SIZE, .use_log = FALSE, + // 0 == follow cache_capacity; see master_config. + .checkpoint_log_size = 0, .num_normal_bg_threads = TEST_CONFIG_DEFAULT_NUM_NORMAL_BG_THREADS, .num_memtable_bg_threads = TEST_CONFIG_DEFAULT_NUM_MEMTABLE_BG_THREADS, .memtable_capacity = MiB_TO_B(TEST_CONFIG_DEFAULT_MEMTABLE_CAPACITY_MB), @@ -153,6 +155,9 @@ config_usage() platform_error_log("\t--no-stats\n"); platform_error_log("\t--log\n"); platform_error_log("\t--no-log\n"); + platform_error_log("\t--checkpoint-log-size-gib\n"); + platform_error_log("\t--checkpoint-log-size-mib\n"); + platform_error_log("\t--checkpoint-log-size-bytes (0 => cache capacity)\n"); platform_error_log("\t--verbose-logging\n"); platform_error_log("\t--no-verbose-logging\n"); platform_error_log("\t--verbose-progress\n"); @@ -384,6 +389,13 @@ config_parse(master_config *cfg, const uint8 num_config, int argc, char *argv[]) config_set_mib("prefetch-budget", cfg, prefetch_budget) {} config_set_gib("prefetch-budget", cfg, prefetch_budget) {} config_set_uint64("prefetch-budget-bytes", cfg, prefetch_budget) {} + config_set_mib("checkpoint-log-size", cfg, checkpoint_log_size) {} + config_set_gib("checkpoint-log-size", cfg, checkpoint_log_size) {} + config_set_uint64("checkpoint-log-size-bytes", + cfg, + checkpoint_log_size) + { + } config_set_mib("memtable-capacity", cfg, memtable_capacity) {} config_set_gib("memtable-capacity", cfg, memtable_capacity) {} config_set_uint64("rough-count-height", cfg, btree_rough_count_height) diff --git a/tests/config.h b/tests/config.h index 089f06a1..3597c1d5 100644 --- a/tests/config.h +++ b/tests/config.h @@ -75,6 +75,14 @@ typedef struct master_config { // log bool32 use_log; + /* + * Take an automatic checkpoint once the live log has grown by this many + * bytes. Zero means follow cache_capacity, matching the production default + * of one cache's worth of log; it is resolved that way at use, so lowering + * --cache-capacity also lowers this unless it is set explicitly. Set it very + * large (UINT64_MAX) to effectively disable automatic checkpoints. + */ + uint64 checkpoint_log_size; // task system uint64 num_normal_bg_threads; // Both bg_threads fields have to be non-zero diff --git a/tests/functional/test.h b/tests/functional/test.h index d107a24c..11bbd1bf 100644 --- a/tests/functional/test.h +++ b/tests/functional/test.h @@ -299,6 +299,11 @@ test_config_init(system_config *system_cfg, // OUT master_cfg->prefetch_budget, master_cfg->use_stats); + // 0 follows the cache size, mirroring the production default. + uint64 checkpoint_log_size = master_cfg->checkpoint_log_size != 0 + ? master_cfg->checkpoint_log_size + : master_cfg->cache_capacity; + rc = core_config_init(&system_cfg->splinter_cfg, &system_cfg->cache_cfg.super, system_cfg->data_cfg, @@ -308,7 +313,7 @@ test_config_init(system_config *system_cfg, // OUT master_cfg->queue_scale_percent, master_cfg->prefetch_budget, master_cfg->use_log, - master_cfg->cache_capacity, + checkpoint_log_size, master_cfg->use_stats, master_cfg->verbose_logging_enabled, master_cfg->log_handle); diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index 0f368cdc..9be3689d 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -492,6 +492,8 @@ CTEST2(splinter, test_auto_checkpoint_on_overwrites) data->system_cfg->splinter_cfg.use_log = TRUE; data->system_cfg->splinter_cfg.checkpoint_log_size_bytes = 2 * data->system_cfg->io_cfg.extent_size; + // Also verify the reported checkpoint count against the internal one. + data->system_cfg->splinter_cfg.use_stats = TRUE; core_handle spl; platform_status rc = core_mkfs(&spl, @@ -536,6 +538,22 @@ CTEST2(splinter, test_auto_checkpoint_on_overwrites) start_generation, memtable_generation(&spl.mt_ctxt)); + /* + * The reported statistic must agree with the machinery's own count: it is + * summed across threads, so this catches both a missed increment and a + * double count. + */ + uint64 reported = 0; + for (threadid thr_i = 0; thr_i < MAX_THREADS; thr_i++) { + reported += spl.stats[thr_i].checkpoints_completed; + } + ASSERT_EQUAL(spl.checkpoint.completions, + reported, + "checkpoints_completed stat (%lu) disagrees with the " + "checkpoint state's count (%lu)\n", + reported, + spl.checkpoint.completions); + // The surviving value must be the last one written. lookup_result qdata; lookup_result_init( From 93c85d42c2e93972120507f95780f03f03be21a2 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Thu, 30 Jul 2026 11:54:57 -0700 Subject: [PATCH 59/64] remove multi-table tests Signed-off-by: Rob Johnson --- test.sh | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/test.sh b/test.sh index a8c213ba..ae0ebdd5 100755 --- a/test.sh +++ b/test.sh @@ -39,19 +39,17 @@ function cache_tests_3() { # 12 minutes function functionality_tests() { # 50 sec each - run driver_test splinter_test --functionality 1000000 100 --seed 135 - run driver_test splinter_test --functionality 1000000 100 --num-normal-bg-threads 4 --num-memtable-bg-threads 2 --seed 135 - run driver_test splinter_test --functionality 1000000 100 --key-size 102 --seed 135 - run driver_test splinter_test --functionality 1000000 100 --key-size 8 --seed 135 - run driver_test splinter_test --functionality 1000000 100 --use-shmem --seed 135 - run driver_test splinter_test --functionality 1000000 100 --use-shmem --num-normal-bg-threads 4 --num-memtable-bg-threads 2 --seed 135 - run driver_test splinter_test --functionality 1000000 100 --use-shmem --key-size 102 --seed 135 - run driver_test splinter_test --functionality 1000000 100 --use-shmem --key-size 8 --seed 135 - run driver_test splinter_test --functionality 1000000 1000 --num-tables 2 --cache-capacity-mib 1024 - run driver_test splinter_test --functionality 1000000 1000 --num-tables 4 --cache-capacity-mib 1024 - run driver_test splinter_test --functionality 1000000 1000 --num-tables 4 --cache-capacity-mib 512 - run driver_test splinter_test --functionality 10000000 1000 --num-tables 1 --cache-capacity-mib 4096 - run driver_test splinter_test --functionality 10000000 1000 --num-tables 2 --cache-capacity-mib 4096 + run driver_test splinter_test --functionality 1000000 100 --seed 135 + run driver_test splinter_test --functionality 1000000 100 --num-normal-bg-threads 4 --num-memtable-bg-threads 2 --seed 135 + run driver_test splinter_test --functionality 1000000 100 --key-size 102 --seed 135 + run driver_test splinter_test --functionality 1000000 100 --key-size 8 --seed 135 + run driver_test splinter_test --functionality 1000000 100 --use-shmem --seed 135 + run driver_test splinter_test --functionality 1000000 100 --use-shmem --num-normal-bg-threads 4 --num-memtable-bg-threads 2 --seed 135 + run driver_test splinter_test --functionality 1000000 100 --use-shmem --key-size 102 --seed 135 + run driver_test splinter_test --functionality 1000000 100 --use-shmem --key-size 8 --seed 135 + run driver_test splinter_test --functionality 1000000 1000 --cache-capacity-mib 1024 + run driver_test splinter_test --functionality 1000000 1000 --cache-capacity-mib 512 + run driver_test splinter_test --functionality 10000000 1000 --cache-capacity-mib 4096 } function parallel_perf_test_1() { From 5c27a42b3a995b45401456f763b11523f42de345 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Thu, 30 Jul 2026 11:55:13 -0700 Subject: [PATCH 60/64] formatting Signed-off-by: Rob Johnson --- src/core.h | 6 +++--- tests/config.c | 5 ++--- tests/config.h | 4 ++-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/core.h b/src/core.h index 27713f47..f4264891 100644 --- a/src/core.h +++ b/src/core.h @@ -98,9 +98,9 @@ typedef struct core_stats { uint64 discarded_deletes; /* - * Checkpoints that ran to completion -- meaning the durable root advanced and - * the retired log's space came back. Counted here for reporting only; the - * authoritative count the checkpoint machinery waits on is + * Checkpoints that ran to completion -- meaning the durable root advanced + * and the retired log's space came back. Counted here for reporting only; + * the authoritative count the checkpoint machinery waits on is * core_checkpoint_state.completions, which must stay monotonic and so is not * affected by core_reset_stats(). */ diff --git a/tests/config.c b/tests/config.c index d9463cda..6123bb1c 100644 --- a/tests/config.c +++ b/tests/config.c @@ -391,9 +391,8 @@ config_parse(master_config *cfg, const uint8 num_config, int argc, char *argv[]) config_set_uint64("prefetch-budget-bytes", cfg, prefetch_budget) {} config_set_mib("checkpoint-log-size", cfg, checkpoint_log_size) {} config_set_gib("checkpoint-log-size", cfg, checkpoint_log_size) {} - config_set_uint64("checkpoint-log-size-bytes", - cfg, - checkpoint_log_size) + config_set_uint64( + "checkpoint-log-size-bytes", cfg, checkpoint_log_size) { } config_set_mib("memtable-capacity", cfg, memtable_capacity) {} diff --git a/tests/config.h b/tests/config.h index 3597c1d5..8fbdb8af 100644 --- a/tests/config.h +++ b/tests/config.h @@ -79,8 +79,8 @@ typedef struct master_config { * Take an automatic checkpoint once the live log has grown by this many * bytes. Zero means follow cache_capacity, matching the production default * of one cache's worth of log; it is resolved that way at use, so lowering - * --cache-capacity also lowers this unless it is set explicitly. Set it very - * large (UINT64_MAX) to effectively disable automatic checkpoints. + * --cache-capacity also lowers this unless it is set explicitly. Set it + * very large (UINT64_MAX) to effectively disable automatic checkpoints. */ uint64 checkpoint_log_size; From 557e29d3a821440406f4fa5e163e5a4cb60117db Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Thu, 30 Jul 2026 12:44:11 -0700 Subject: [PATCH 61/64] remove multi-table testing code Signed-off-by: Rob Johnson --- test.sh | 1 - tests/functional/splinter_test.c | 1491 +++++++++---------------- tests/functional/test_functionality.c | 234 ++-- tests/functional/test_functionality.h | 4 +- tests/unit/splinter_test.c | 27 +- 5 files changed, 624 insertions(+), 1133 deletions(-) diff --git a/test.sh b/test.sh index ae0ebdd5..2ad9ad23 100755 --- a/test.sh +++ b/test.sh @@ -191,7 +191,6 @@ function all_tests() { } function main() { - echo > db.sizes.log if [ -z "$TESTS_FUNCTION" ]; then TESTS_FUNCTION="all_tests" fi diff --git a/tests/functional/splinter_test.c b/tests/functional/splinter_test.c index b8e641e2..9cb88732 100644 --- a/tests/functional/splinter_test.c +++ b/tests/functional/splinter_test.c @@ -51,10 +51,9 @@ typedef struct test_splinter_thread_params { platform_thread thread; core_handle *spl; test_config *test_cfg; - uint64 *total_ops; + uint64 total_ops; uint64 *curr_op; uint64 op_granularity; - uint8 num_tables; uint64 thread_number; task_system *ts; platform_status rc; @@ -63,7 +62,7 @@ typedef struct test_splinter_thread_params { stats_insert insert_stats; uint64 num_ops_per_thread[NUM_OP_TYPES]; // in each round bool32 expected_found; - test_async_lookup *async_lookup[8]; // async lookup state per table + test_async_lookup *async_lookup; uint64 insert_rate; stats_lookup lookup_stats[NUM_LOOKUP_TYPES]; uint8 lookup_positive_pct; // parallel lookup positive % @@ -91,24 +90,6 @@ typedef struct trunk_range_perf_params { */ typedef void (*test_trunk_thread_hdlr)(void *arg); -static inline bool32 -test_is_done(const uint8 done, const uint8 n) -{ - return (((done >> n) & 1) != 0); -} - -static inline void -test_set_done(uint8 *done, const uint8 n) -{ - *done |= 1 << n; -} - -static inline bool32 -test_all_done(const uint8 done, const uint8 num_tables) -{ - return (done == ((1 << num_tables) - 1)); -} - /* * test_trunk_insert_thread() -- Per-thread function to drive inserts. */ @@ -117,17 +98,13 @@ test_trunk_insert_thread(void *arg) { test_splinter_thread_params *params = (test_splinter_thread_params *)arg; - core_handle *spl_tables = params->spl; + core_handle *spl = params->spl; const test_config *test_cfg = params->test_cfg; - const uint64 *total_ops = params->total_ops; + const uint64 total_ops = params->total_ops; uint64 *curr_op = params->curr_op; uint64 op_granularity = params->op_granularity; uint64 thread_number = params->thread_number; - uint8 num_tables = params->num_tables; platform_heap_id heap_id = platform_get_heap_id(); - platform_assert(num_tables <= 8); - uint64 *insert_base = TYPED_ARRAY_ZALLOC(heap_id, insert_base, num_tables); - uint8 done = 0; uint64 num_inserts = 0; timestamp next_check_time = platform_get_timestamp(); @@ -136,76 +113,60 @@ test_trunk_insert_thread(void *arg) merge_accumulator msg; merge_accumulator_init(&msg, heap_id); + uint64 insert_base = 0; while (1) { - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - if (test_is_done(done, spl_idx)) { - continue; - } - test_print_progress(¶ms->progress, - insert_base[spl_idx] / (total_ops[spl_idx] / 100), - PLATFORM_CR "Thread %lu inserting %3lu%% " - "complete for table %u", - thread_number, - insert_base[spl_idx] / (total_ops[spl_idx] / 100), - spl_idx); - insert_base[spl_idx] = - __sync_fetch_and_add(&curr_op[spl_idx], op_granularity); - if (insert_base[spl_idx] >= total_ops[spl_idx]) { - test_set_done(&done, spl_idx); - } - if (test_all_done(done, num_tables)) { - platform_default_log(" Test done for all %d tables.\n", num_tables); - goto out; - } + test_print_progress(¶ms->progress, + insert_base / (total_ops / 100), + PLATFORM_CR "Thread %lu inserting %3lu%% complete", + thread_number, + insert_base / (total_ops / 100)); + insert_base = __sync_fetch_and_add(curr_op, op_granularity); + if (insert_base >= total_ops) { + platform_default_log(" Test done.\n"); + goto out; } DECLARE_AUTO_KEY_BUFFER(keybuf, heap_id); for (uint64 op_offset = 0; op_offset != op_granularity; op_offset++) { - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - uint64 insert_num = insert_base[spl_idx] + op_offset; - if (test_is_done(done, spl_idx)) { - continue; - } - core_handle *spl = &spl_tables[spl_idx]; + uint64 insert_num = insert_base + op_offset; - timestamp ts; - if (spl->cfg.use_stats) { - ts = platform_get_timestamp(); - } - test_key(&keybuf, - test_cfg[spl_idx].key_type, - insert_num, - thread_number, - test_cfg[spl_idx].semiseq_freq, - test_cfg[spl_idx].key_size, - test_cfg[spl_idx].period); - generate_test_message(test_cfg->gen, insert_num, &msg); - platform_status rc = core_insert(spl, - key_buffer_key(&keybuf), - merge_accumulator_to_message(&msg), - NULL); - platform_assert_status_ok(rc); - if (spl->cfg.use_stats) { - ts = platform_timestamp_elapsed(ts); - if (ts > params->insert_stats.latency_max) { - params->insert_stats.latency_max = ts; - } + timestamp ts; + if (spl->cfg.use_stats) { + ts = platform_get_timestamp(); + } + test_key(&keybuf, + test_cfg->key_type, + insert_num, + thread_number, + test_cfg->semiseq_freq, + test_cfg->key_size, + test_cfg->period); + generate_test_message(test_cfg->gen, insert_num, &msg); + platform_status rc = core_insert(spl, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); + platform_assert_status_ok(rc); + if (spl->cfg.use_stats) { + ts = platform_timestamp_elapsed(ts); + if (ts > params->insert_stats.latency_max) { + params->insert_stats.latency_max = ts; } + } - // Throttle thread based on insert rate if needed. - if (insert_rate != 0) { - timestamp now = platform_get_timestamp(); - if (now <= next_check_time) { - num_inserts++; - if (num_inserts >= insert_rate) { - platform_sleep_ns(next_check_time - now); - } - } else { - // reset and check again after 10 msec. - num_inserts = 0; - next_check_time = now + (USEC_TO_NSEC(10000)); + // Throttle thread based on insert rate if needed. + if (insert_rate != 0) { + timestamp now = platform_get_timestamp(); + if (now <= next_check_time) { + num_inserts++; + if (num_inserts >= insert_rate) { + platform_sleep_ns(next_check_time - now); } + } else { + // reset and check again after 10 msec. + num_inserts = 0; + next_check_time = now + (USEC_TO_NSEC(10000)); } } } @@ -213,11 +174,7 @@ test_trunk_insert_thread(void *arg) out: merge_accumulator_deinit(&msg); params->rc = STATUS_OK; - platform_free(platform_get_heap_id(), insert_base); - for (uint64 i = 0; i < num_tables; i++) { - core_handle *spl = &spl_tables[i]; - core_perform_tasks(spl); - } + core_perform_tasks(spl); } /* @@ -228,118 +185,88 @@ test_trunk_lookup_thread(void *arg) { test_splinter_thread_params *params = (test_splinter_thread_params *)arg; - core_handle *spl_tables = params->spl; + core_handle *spl = params->spl; const test_config *test_cfg = params->test_cfg; - const uint64 *total_ops = params->total_ops; + const uint64 total_ops = params->total_ops; uint64 *curr_op = params->curr_op; uint64 op_granularity = params->op_granularity; uint64 thread_number = params->thread_number; bool32 expected_found = params->expected_found; - uint8 num_tables = params->num_tables; + test_async_lookup *async_lookup = params->async_lookup; verify_tuple_arg vtarg = {.expected_found = expected_found, .stats = ¶ms->lookup_stats[ASYNC_LU]}; platform_heap_id heap_id = platform_get_heap_id(); - platform_assert(num_tables <= 8); - uint64 *lookup_base = TYPED_ARRAY_ZALLOC(heap_id, lookup_base, num_tables); - uint8 done = 0; - lookup_result data; lookup_result_init(&data, NULL, SPLINTERDB_LOOKUP_VALUE, 0, NULL); + uint64 lookup_base = 0; while (1) { - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - if (test_is_done(done, spl_idx)) { - continue; - } - test_print_progress(¶ms->progress, - lookup_base[spl_idx] / (total_ops[spl_idx] / 100), - PLATFORM_CR "Thread %lu lookups %3lu%% complete " - "for table %u", - thread_number, - lookup_base[spl_idx] / (total_ops[spl_idx] / 100), - spl_idx); - lookup_base[spl_idx] = - __sync_fetch_and_add(&curr_op[spl_idx], op_granularity); - if (lookup_base[spl_idx] >= total_ops[spl_idx]) { - test_set_done(&done, spl_idx); - } - if (test_all_done(done, num_tables)) { - goto out; - } + test_print_progress(¶ms->progress, + lookup_base / (total_ops / 100), + PLATFORM_CR "Thread %lu lookups %3lu%% complete", + thread_number, + lookup_base / (total_ops / 100)); + lookup_base = __sync_fetch_and_add(curr_op, op_granularity); + if (lookup_base >= total_ops) { + goto out; } DECLARE_AUTO_KEY_BUFFER(keybuf, heap_id); for (uint64 op_offset = 0; op_offset != op_granularity; op_offset++) { - uint8 spl_idx; - for (spl_idx = 0; spl_idx < num_tables; spl_idx++) { - if (test_is_done(done, spl_idx)) { - continue; - } - core_handle *spl = &spl_tables[spl_idx]; - test_async_lookup *async_lookup = params->async_lookup[spl_idx]; - test_async_ctxt *ctxt; - uint64 lookup_num = lookup_base[spl_idx] + op_offset; - timestamp ts; - - if (async_lookup->max_async_inflight == 0) { - platform_status rc; - - test_key(&keybuf, - test_cfg[spl_idx].key_type, - lookup_num, - thread_number, - test_cfg[spl_idx].semiseq_freq, - test_cfg[spl_idx].key_size, - test_cfg[spl_idx].period); - ts = platform_get_timestamp(); - lookup_result_set_data_config(&data, spl->cfg.data_cfg); - rc = core_lookup(spl, key_buffer_key(&keybuf), &data); - ts = platform_timestamp_elapsed(ts); - if (ts > params->lookup_stats[SYNC_LU].latency_max) { - params->lookup_stats[SYNC_LU].latency_max = ts; - } - platform_assert(SUCCESS(rc)); - verify_tuple(spl, - test_cfg->gen, - lookup_num, - key_buffer_key(&keybuf), - merge_accumulator_to_message( - lookup_result_accumulator(&data)), - expected_found); - } else { - ctxt = test_async_ctxt_get(spl, async_lookup, &vtarg); - test_key(&ctxt->key, - test_cfg[spl_idx].key_type, - lookup_num, - thread_number, - test_cfg[spl_idx].semiseq_freq, - test_cfg[spl_idx].key_size, - test_cfg[spl_idx].period); - ctxt->lookup_num = lookup_num; - async_ctxt_submit(spl, - async_lookup, - ctxt, - ¶ms->lookup_stats[ASYNC_LU].latency_max, - verify_tuple_callback, - &vtarg); + test_async_ctxt *ctxt; + uint64 lookup_num = lookup_base + op_offset; + timestamp ts; + + if (async_lookup->max_async_inflight == 0) { + platform_status rc; + + test_key(&keybuf, + test_cfg->key_type, + lookup_num, + thread_number, + test_cfg->semiseq_freq, + test_cfg->key_size, + test_cfg->period); + ts = platform_get_timestamp(); + lookup_result_set_data_config(&data, spl->cfg.data_cfg); + rc = core_lookup(spl, key_buffer_key(&keybuf), &data); + ts = platform_timestamp_elapsed(ts); + if (ts > params->lookup_stats[SYNC_LU].latency_max) { + params->lookup_stats[SYNC_LU].latency_max = ts; } + platform_assert(SUCCESS(rc)); + verify_tuple( + spl, + test_cfg->gen, + lookup_num, + key_buffer_key(&keybuf), + merge_accumulator_to_message(lookup_result_accumulator(&data)), + expected_found); + } else { + ctxt = test_async_ctxt_get(spl, async_lookup, &vtarg); + test_key(&ctxt->key, + test_cfg->key_type, + lookup_num, + thread_number, + test_cfg->semiseq_freq, + test_cfg->key_size, + test_cfg->period); + ctxt->lookup_num = lookup_num; + async_ctxt_submit(spl, + async_lookup, + ctxt, + ¶ms->lookup_stats[ASYNC_LU].latency_max, + verify_tuple_callback, + &vtarg); } } - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - if (test_is_done(done, spl_idx)) { - continue; - } - core_handle *spl = &spl_tables[spl_idx]; - test_async_lookup *async_lookup = params->async_lookup[spl_idx]; - test_wait_for_inflight(spl, async_lookup, &vtarg); - } + test_wait_for_inflight(spl, async_lookup, &vtarg); } out: lookup_result_deinit(&data); params->rc = STATUS_OK; - platform_free(platform_get_heap_id(), lookup_base); } static void @@ -360,21 +287,16 @@ test_trunk_range_thread(void *arg) { test_splinter_thread_params *params = (test_splinter_thread_params *)arg; - core_handle *spl_tables = params->spl; + core_handle *spl = params->spl; const test_config *test_cfg = params->test_cfg; - const uint64 *total_ops = params->total_ops; + const uint64 total_ops = params->total_ops; uint64 *curr_op = params->curr_op; uint64 op_granularity = params->op_granularity; uint64 thread_number = params->thread_number; uint64 min_range_length = params->min_range_length; uint64 max_range_length = params->max_range_length; - uint8 num_tables = params->num_tables; platform_heap_id heap_id = platform_get_heap_id(); - platform_assert(num_tables <= 8); - uint64 *range_base = TYPED_ARRAY_ZALLOC(heap_id, range_base, num_tables); - uint8 done = 0; - bool32 verbose_progress = test_show_verbose_progress(test_cfg->test_exec_cfg); uint64 test_start_time = platform_get_timestamp(); @@ -383,83 +305,65 @@ test_trunk_range_thread(void *arg) DECLARE_AUTO_KEY_BUFFER(start_key, heap_id); + uint64 range_base = 0; while (1) { - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - if (test_is_done(done, spl_idx)) { - continue; - } - int pct_done = (range_base[spl_idx] / (total_ops[spl_idx] / 100)); - char *newmsg = ""; - - if (verbose_progress) { - // For all threads other than thread 0, periodically emit a message - if (thread_number && pct_done && ((pct_done % 10) == 0)) { - uint64 delta_time = platform_timestamp_elapsed(start_time); - uint64 total_time = platform_timestamp_elapsed(test_start_time); - snprintf(progress_msg, - sizeof(progress_msg), - " ... delta=%lus, total time=%lus\n", - NSEC_TO_SEC(delta_time), - NSEC_TO_SEC(total_time)); - newmsg = progress_msg; - - // Reset ... for next batch of progress messages - start_time = platform_get_timestamp(); - } - } - test_print_progress(¶ms->progress, - range_base[spl_idx] / (total_ops[spl_idx] / 100), - PLATFORM_CR - "Thread %lu range lookups %3lu%% " - "complete for table %u, range_base=%lu%s", - thread_number, - range_base[spl_idx] / (total_ops[spl_idx] / 100), - spl_idx, - range_base[spl_idx], - newmsg); - - range_base[spl_idx] = - __sync_fetch_and_add(&curr_op[spl_idx], op_granularity); - if (range_base[spl_idx] >= total_ops[spl_idx]) { - test_set_done(&done, spl_idx); - } - if (test_all_done(done, num_tables)) { - goto out; + uint64 pct_done = (range_base / (total_ops / 100)); + char *newmsg = ""; + + if (verbose_progress) { + // For all threads other than thread 0, periodically emit a message + if (thread_number && pct_done && ((pct_done % 10) == 0)) { + uint64 delta_time = platform_timestamp_elapsed(start_time); + uint64 total_time = platform_timestamp_elapsed(test_start_time); + snprintf(progress_msg, + sizeof(progress_msg), + " ... delta=%lus, total time=%lus\n", + NSEC_TO_SEC(delta_time), + NSEC_TO_SEC(total_time)); + newmsg = progress_msg; + + // Reset ... for next batch of progress messages + start_time = platform_get_timestamp(); } } + test_print_progress(¶ms->progress, + pct_done, + PLATFORM_CR "Thread %lu range lookups %3lu%% " + "complete, range_base=%lu%s", + thread_number, + pct_done, + range_base, + newmsg); + + range_base = __sync_fetch_and_add(curr_op, op_granularity); + if (range_base >= total_ops) { + goto out; + } for (uint64 op_offset = 0; op_offset != op_granularity; op_offset++) { - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - if (test_is_done(done, spl_idx)) { - continue; - } - core_handle *spl = &spl_tables[spl_idx]; + uint64 range_num = range_base + op_offset; + test_key(&start_key, + test_cfg->key_type, + range_num, + thread_number, + test_cfg->semiseq_freq, + test_cfg->key_size, + test_cfg->period); + uint64 range_tuples = + test_range(range_num, min_range_length, max_range_length); + platform_status rc = + core_apply_to_range(spl, + key_buffer_key(&start_key), + range_tuples, + count_range_tuple_func, + ¶ms->range_tuples_returned); + platform_assert_status_ok(rc); - uint64 range_num = range_base[spl_idx] + op_offset; - test_key(&start_key, - test_cfg[spl_idx].key_type, - range_num, - thread_number, - test_cfg[spl_idx].semiseq_freq, - test_cfg[spl_idx].key_size, - test_cfg[spl_idx].period); - uint64 range_tuples = - test_range(range_num, min_range_length, max_range_length); - platform_status rc = - core_apply_to_range(spl, - key_buffer_key(&start_key), - range_tuples, - count_range_tuple_func, - ¶ms->range_tuples_returned); - platform_assert_status_ok(rc); - - params->range_lookups_done++; - } + params->range_lookups_done++; } } out: params->rc = STATUS_OK; - platform_free(platform_get_heap_id(), range_base); } /* @@ -477,78 +381,59 @@ static bool32 advance_base(test_splinter_thread_params *params, uint64 *curr_op, uint64 *base, - uint8 *done, random_state *rs, test_splinter_pthread_op_type type) { - const uint64 *total_ops = params->total_ops; - const uint64 op_granularity = params->op_granularity; - const uint8 num_tables = params->num_tables; - const uint8 lookup_positive_pct = params->lookup_positive_pct; - - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - if (test_is_done(*done, spl_idx)) { - continue; + const uint64 total_ops = params->total_ops; + const uint64 op_granularity = params->op_granularity; + const uint8 lookup_positive_pct = params->lookup_positive_pct; + + if (type == OP_INSERT) { + test_print_progress(¶ms->progress, + *base / (total_ops / 100), + PLATFORM_CR "inserting/lookups %3lu%% complete", + *base / (total_ops / 100)); + *base = __sync_fetch_and_add(curr_op, op_granularity); + if (*base >= total_ops) { + return TRUE; } - - if (type == OP_INSERT) { - test_print_progress(¶ms->progress, - base[spl_idx] / (total_ops[spl_idx] / 100), - PLATFORM_CR "inserting/lookups %3lu%% complete " - "for table %u", - base[spl_idx] / (total_ops[spl_idx] / 100), - spl_idx); - base[spl_idx] = - __sync_fetch_and_add(&curr_op[spl_idx], op_granularity); - if (base[spl_idx] >= total_ops[spl_idx]) { - test_set_done(done, spl_idx); - } - if (test_all_done(*done, num_tables)) { - return TRUE; + } else { + /* lookup */ + uint64 random_base; + uint64 local_curr_op = *curr_op; + // pick an random number to determine + // if positive or negative lookup + // [0, lookup_positive_pct) is positive lookup + if (random_next_uint64(rs) % 100 < lookup_positive_pct) { + if (local_curr_op == op_granularity) { + random_base = 0; + } else { + // positive lookup by selecting random base + // [op_granularity, local_curr_op) + random_base = + ((random_next_uint64(rs) % (local_curr_op / op_granularity - 1)) + + 1) + * op_granularity; } } else { - /* lookup */ - uint64 random_base; - uint64 local_curr_op = curr_op[spl_idx]; - // pick an random number to determine - // if positive or negative lookup - // [0, lookup_positive_pct) is positive lookup - if (random_next_uint64(rs) % 100 < lookup_positive_pct) { - if (local_curr_op == op_granularity) { - random_base = 0; - } else { - // positive lookup by selecting random base - // [op_granularity, local_curr_op) - random_base = ((random_next_uint64(rs) - % (local_curr_op / op_granularity - 1)) - + 1) - * op_granularity; - } + if (local_curr_op >= total_ops - op_granularity) { + // one interval left, + // may lookup the key never be inserted + random_base = + (random_next_uint64(rs) / op_granularity) * op_granularity + + local_curr_op; } else { - if (local_curr_op >= total_ops[spl_idx] - op_granularity) { - // one interval left, - // may lookup the key never be inserted - random_base = - (random_next_uint64(rs) / op_granularity) * op_granularity - + local_curr_op; - } else { - // negative lookup by select random base - // [local_curr_op + op_granularity, total_ops - op_granularity] - random_base = - ((random_next_uint64(rs) - % ((total_ops[spl_idx] - local_curr_op) / op_granularity - - 1)) - + 1) - * op_granularity - + local_curr_op; - } + // negative lookup by select random base + // [local_curr_op + op_granularity, total_ops - op_granularity] + random_base = ((random_next_uint64(rs) + % ((total_ops - local_curr_op) / op_granularity + - 1)) + + 1) + * op_granularity + + local_curr_op; } - base[spl_idx] = random_base; } - } - - if (test_all_done(*done, num_tables)) { - return TRUE; + *base = random_base; } return FALSE; @@ -564,17 +449,15 @@ advance_base(test_splinter_thread_params *params, */ static void do_operation(test_splinter_thread_params *params, - const uint64 *base, + uint64 base, uint64 num_ops, uint64 op_offset, - const uint8 *done, bool32 is_insert) { - core_handle *spl_tables = params->spl; + core_handle *spl = params->spl; const test_config *test_cfg = params->test_cfg; uint64 op_granularity = params->op_granularity; uint64 thread_number = params->thread_number; - uint8 num_tables = params->num_tables; verify_tuple_arg vtarg = {.stats_only = TRUE, .stats = ¶ms->lookup_stats[ASYNC_LU]}; platform_heap_id heap_id = platform_get_heap_id(); @@ -587,82 +470,76 @@ do_operation(test_splinter_thread_params *params, for (uint64 op_idx = op_offset; op_idx != op_offset + num_ops; op_idx++) { if (op_idx >= op_granularity) { - return; + break; } - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - if (test_is_done(*done, spl_idx)) { - continue; + uint64 op_num = base + op_idx; + timestamp ts; + + if (is_insert) { + test_key(&keybuf, + test_cfg->key_type, + op_num, + thread_number, + test_cfg->semiseq_freq, + test_cfg->key_size, + test_cfg->period); + generate_test_message(test_cfg->gen, op_num, &msg); + ts = platform_get_timestamp(); + platform_status rc = core_insert(spl, + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); + platform_assert_status_ok(rc); + ts = platform_timestamp_elapsed(ts); + params->insert_stats.duration += ts; + if (ts > params->insert_stats.latency_max) { + params->insert_stats.latency_max = ts; } - core_handle *spl = &spl_tables[spl_idx]; - uint64 op_num = base[spl_idx] + op_idx; - timestamp ts; + } else { + test_async_lookup *async_lookup = params->async_lookup; + test_async_ctxt *ctxt; + + if (async_lookup->max_async_inflight == 0) { + platform_status rc; - if (is_insert) { test_key(&keybuf, - test_cfg[spl_idx].key_type, + test_cfg->key_type, op_num, thread_number, - test_cfg[spl_idx].semiseq_freq, - test_cfg[spl_idx].key_size, - test_cfg[spl_idx].period); - generate_test_message(test_cfg->gen, op_num, &msg); - ts = platform_get_timestamp(); - platform_status rc = core_insert(spl, - key_buffer_key(&keybuf), - merge_accumulator_to_message(&msg), - NULL); - platform_assert_status_ok(rc); + test_cfg->semiseq_freq, + test_cfg->key_size, + test_cfg->period); + ts = platform_get_timestamp(); + lookup_result_set_data_config(&lookup, spl->cfg.data_cfg); + rc = core_lookup(spl, key_buffer_key(&keybuf), &lookup); + platform_assert(SUCCESS(rc)); ts = platform_timestamp_elapsed(ts); - params->insert_stats.duration += ts; - if (ts > params->insert_stats.latency_max) { - params->insert_stats.latency_max = ts; + if (ts > params->lookup_stats[SYNC_LU].latency_max) { + params->lookup_stats[SYNC_LU].latency_max = ts; } - } else { - test_async_lookup *async_lookup = params->async_lookup[spl_idx]; - test_async_ctxt *ctxt; - - if (async_lookup->max_async_inflight == 0) { - platform_status rc; - - test_key(&keybuf, - test_cfg[spl_idx].key_type, - op_num, - thread_number, - test_cfg[spl_idx].semiseq_freq, - test_cfg[spl_idx].key_size, - test_cfg[spl_idx].period); - ts = platform_get_timestamp(); - lookup_result_set_data_config(&lookup, spl->cfg.data_cfg); - rc = core_lookup(spl, key_buffer_key(&keybuf), &lookup); - platform_assert(SUCCESS(rc)); - ts = platform_timestamp_elapsed(ts); - if (ts > params->lookup_stats[SYNC_LU].latency_max) { - params->lookup_stats[SYNC_LU].latency_max = ts; - } - bool32 found = lookup_result_found(&lookup); - if (found) { - params->lookup_stats[SYNC_LU].num_found++; - } else { - params->lookup_stats[SYNC_LU].num_not_found++; - } + bool32 found = lookup_result_found(&lookup); + if (found) { + params->lookup_stats[SYNC_LU].num_found++; } else { - ctxt = test_async_ctxt_get(spl, async_lookup, &vtarg); - test_key(&ctxt->key, - test_cfg[spl_idx].key_type, - op_num, - thread_number, - test_cfg[spl_idx].semiseq_freq, - test_cfg[spl_idx].key_size, - test_cfg[spl_idx].period); - ctxt->lookup_num = op_num; - async_ctxt_submit(spl, - async_lookup, - ctxt, - ¶ms->lookup_stats[ASYNC_LU].latency_max, - verify_tuple_callback, - &vtarg); + params->lookup_stats[SYNC_LU].num_not_found++; } + } else { + ctxt = test_async_ctxt_get(spl, async_lookup, &vtarg); + test_key(&ctxt->key, + test_cfg->key_type, + op_num, + thread_number, + test_cfg->semiseq_freq, + test_cfg->key_size, + test_cfg->period); + ctxt->lookup_num = op_num; + async_ctxt_submit(spl, + async_lookup, + ctxt, + ¶ms->lookup_stats[ASYNC_LU].latency_max, + verify_tuple_callback, + &vtarg); } } } @@ -686,31 +563,27 @@ test_trunk_insert_lookup_thread(void *arg) { test_splinter_thread_params *params = (test_splinter_thread_params *)arg; - core_handle *spl_tables = params->spl; - uint8 num_tables = params->num_tables; - uint64 op_granularity = params->op_granularity; - uint64 seed = params->seed; + core_handle *spl = params->spl; + uint64 op_granularity = params->op_granularity; + uint64 seed = params->seed; + verify_tuple_arg vtarg = {.stats_only = TRUE, + .stats = ¶ms->lookup_stats[ASYNC_LU]}; - platform_assert(num_tables <= 8); - - uint64 *bases[NUM_OP_TYPES]; + uint64 bases[NUM_OP_TYPES]; uint64 granularities[NUM_OP_TYPES]; uint64 offsets[NUM_OP_TYPES]; - uint8 insert_done = 0; - uint64 num_ops = 0; + uint64 num_ops = 0; random_state rs; random_init(&rs, seed, 0); for (uint8 i = 0; i < NUM_OP_TYPES; i++) { - bases[i] = - TYPED_ARRAY_ZALLOC(platform_get_heap_id(), bases[i], num_tables); - + bases[i] = 0; granularities[i] = params->num_ops_per_thread[i]; offsets[i] = 0; } for (uint8 i = 0; i < NUM_OP_TYPES; i++) { - advance_base(params, params->curr_op, bases[i], &insert_done, &rs, i); + advance_base(params, params->curr_op, &bases[i], &rs, i); } while (1) { @@ -730,7 +603,6 @@ test_trunk_insert_lookup_thread(void *arg) bases[op_type], num_ops, offsets[op_type], - &insert_done, op_type == OP_INSERT); offsets[op_type] += num_ops; } @@ -739,12 +611,8 @@ test_trunk_insert_lookup_thread(void *arg) || offsets[op_type] >= op_granularity) { num_ops = granularities[op_type] - num_ops; - if (advance_base(params, - params->curr_op, - bases[op_type], - &insert_done, - &rs, - op_type)) + if (advance_base( + params, params->curr_op, &bases[op_type], &rs, op_type)) { goto out; } @@ -755,7 +623,6 @@ test_trunk_insert_lookup_thread(void *arg) bases[op_type], num_ops, offsets[op_type], - &insert_done, op_type == OP_INSERT); offsets[op_type] += num_ops; } @@ -764,116 +631,76 @@ test_trunk_insert_lookup_thread(void *arg) } out: - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - verify_tuple_arg vtarg = {.stats_only = TRUE, - .stats = ¶ms->lookup_stats[ASYNC_LU]}; - test_async_lookup *async_lookup = params->async_lookup[spl_idx]; - test_wait_for_inflight(spl, async_lookup, &vtarg); - } + test_wait_for_inflight(spl, params->async_lookup, &vtarg); params->rc = STATUS_OK; - for (uint8 i = 0; i < NUM_OP_TYPES; i++) { - platform_free(platform_get_heap_id(), bases[i]); - } } +/* + * The unified superblock is per-device: core owns one superblock per device, so + * these tests drive exactly one table. + */ static platform_status -test_trunk_create_tables(core_handle **spl_handles, - system_config *cfg, - allocator *al, - cache *cc[], - task_system *ts, - platform_heap_id hid, - uint8 num_tables, - uint8 num_caches) +test_trunk_create_table(core_handle **spl_handle, + system_config *cfg, + allocator *al, + cache *cc, + task_system *ts, + platform_heap_id hid) { - /* - * The unified superblock is per-device: core owns one superblock per - * device, so multiple tables sharing one device/allocator is not supported - * until the multi-tree machinery lands (the on-disk format already reserves - * room for it). See the "core owns; 1 table/device" design decision. - */ - platform_assert(num_tables == 1, - "This test currently supports a single table per device " - "(num_tables=%u); rerun with --num-tables 1.", - num_tables); - // These functional tests use rc_allocator exclusively; core_mkfs needs the // io handle, which the allocator holds. io_handle *io = ((rc_allocator *)al)->io; - core_handle *spl_tables = TYPED_ARRAY_ZALLOC(hid, spl_tables, num_tables); - if (spl_tables == NULL) { + core_handle *spl = TYPED_ZALLOC(hid, spl); + if (spl == NULL) { return STATUS_NO_MEMORY; } - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - cache *cache_to_use = num_caches > 1 ? cc[spl_idx] : *cc; - platform_status rc = core_mkfs(&spl_tables[spl_idx], - &cfg[spl_idx].splinter_cfg, - al, - cache_to_use, - io, - ts, - test_generate_allocator_root_id(), - hid); - if (!SUCCESS(rc)) { - for (uint8 del_idx = 0; del_idx < spl_idx; del_idx++) { - core_destroy(&spl_tables[del_idx]); - } - platform_free(hid, spl_tables); - return rc; - } + platform_status rc = core_mkfs(spl, + &cfg->splinter_cfg, + al, + cc, + io, + ts, + test_generate_allocator_root_id(), + hid); + if (!SUCCESS(rc)) { + platform_free(hid, spl); + return rc; } - *spl_handles = spl_tables; + *spl_handle = spl; return STATUS_OK; } static void -test_trunk_destroy_tables(core_handle *spl_tables, - platform_heap_id hid, - uint8 num_tables) +test_trunk_destroy_table(core_handle *spl, platform_heap_id hid) { - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_destroy(&spl_tables[spl_idx]); - } - platform_free(hid, spl_tables); + core_destroy(spl); + platform_free(hid, spl); } /* - * compute_per_table_inserts() -- + * compute_num_inserts() -- * - * Helper function to compute the # of inserts for each table, based on the + * Helper function to compute the # of inserts for the table, based on the * configuration specified. - * - * Returns: Total # of inserts to-be-done in the workload */ static uint64 -compute_per_table_inserts(uint64 *per_table_inserts, // OUT - system_config *cfg, // IN - test_config *test_cfg, // IN - uint8 num_tables) +compute_num_inserts(test_config *test_cfg) // IN { - uint64 tuple_size; - uint64 num_inserts; uint64 num_inserts_cfg = test_cfg->test_exec_cfg->num_inserts; - uint64 total_inserts = 0; - - for (uint8 i = 0; i < num_tables; i++) { - tuple_size = - test_cfg[i].key_size + generator_average_message_size(test_cfg->gen); - num_inserts = (num_inserts_cfg ? num_inserts_cfg - : test_cfg[i].tree_size / tuple_size); - if (test_cfg[i].key_type == TEST_PERIODIC) { - test_cfg[i].period = num_inserts; - num_inserts *= test_cfg[i].num_periods; - } - per_table_inserts[i] = ROUNDUP(num_inserts, TEST_INSERT_GRANULARITY); - total_inserts += per_table_inserts[i]; + uint64 tuple_size = + test_cfg->key_size + generator_average_message_size(test_cfg->gen); + uint64 num_inserts = + (num_inserts_cfg ? num_inserts_cfg : test_cfg->tree_size / tuple_size); + + if (test_cfg->key_type == TEST_PERIODIC) { + test_cfg->period = num_inserts; + num_inserts *= test_cfg->num_periods; } - return total_inserts; + return ROUNDUP(num_inserts, TEST_INSERT_GRANULARITY); } /* @@ -883,11 +710,10 @@ compute_per_table_inserts(uint64 *per_table_inserts, // OUT */ static void load_thread_params(test_splinter_thread_params *params, - core_handle *spl_tables, + core_handle *spl, test_config *test_cfg, - uint64 *per_table_inserts, + uint64 num_inserts, uint64 *curr_op, - uint8 num_tables, task_system *ts, uint64 insert_rate, uint64 num_insert_threads, @@ -895,13 +721,12 @@ load_thread_params(test_splinter_thread_params *params, bool32 is_parallel) { for (uint64 i = 0; i < num_threads; i++) { - params[i].spl = spl_tables; + params[i].spl = spl; params[i].test_cfg = test_cfg; - params[i].total_ops = per_table_inserts; + params[i].total_ops = num_inserts; params[i].curr_op = curr_op; params[i].op_granularity = TEST_INSERT_GRANULARITY; params[i].thread_number = i; - params[i].num_tables = num_tables; params[i].ts = ts; if (!is_parallel) { @@ -939,42 +764,34 @@ do_n_thread_creates(const char *thread_type, /* * do_n_async_ctxt_inits() -- * - * Helper function to setup Async contexts for num_threads threads, each thread - * processing num_tables tables. This async-lookup context is tied to each - * table. Lookup execution for each table needs to access this context before it - * can check if (max_async_inflight == 0). + * Helper function to setup Async contexts for num_threads threads. Lookup + * execution needs to access this context before it can check if + * (max_async_inflight == 0). */ static void do_n_async_ctxt_inits(platform_heap_id hid, uint64 num_threads, - uint8 num_tables, uint64 max_async_inflight, - system_config *cfg, test_splinter_thread_params *params) { for (uint64 i = 0; i < num_threads; i++) { - for (uint8 j = 0; j < num_tables; j++) { - async_ctxt_init(hid, max_async_inflight, ¶ms[i].async_lookup[j]); - } + async_ctxt_init(hid, max_async_inflight, ¶ms[i].async_lookup); } } /* * do_n_async_ctxt_deinits() -- * - * Helper function to dismantle n-Async contexts for m-tables. + * Helper function to dismantle n-Async contexts. */ static void do_n_async_ctxt_deinits(platform_heap_id hid, uint64 num_threads, - uint8 num_tables, test_splinter_thread_params *params) { for (uint64 i = 0; i < num_threads; i++) { - for (uint8 j = 0; j < num_tables; j++) { - async_ctxt_deinit(hid, params[i].async_lookup[j]); - params[i].async_lookup[j] = NULL; - } + async_ctxt_deinit(hid, params[i].async_lookup); + params[i].async_lookup = NULL; } } @@ -987,23 +804,19 @@ do_n_async_ctxt_deinits(platform_heap_id hid, */ static platform_status splinter_perf_inserts(platform_heap_id hid, - system_config *cfg, test_config *test_cfg, - core_handle *spl_tables, - cache *cc[], + core_handle *spl, + cache *cc, task_system *ts, test_splinter_thread_params *params, - uint64 *per_table_inserts, uint64 *curr_op, uint64 num_insert_threads, uint64 num_threads, - uint8 num_tables, uint64 insert_rate, uint64 *total_inserts) { platform_status rc; - *total_inserts = - compute_per_table_inserts(per_table_inserts, cfg, test_cfg, num_tables); + *total_inserts = compute_num_inserts(test_cfg); platform_default_log("%s() starting num_insert_threads=%lu, num_threads=%lu" ", num_inserts=%lu (~%lu million) ...\n", @@ -1014,11 +827,10 @@ splinter_perf_inserts(platform_heap_id hid, (*total_inserts / MILLION)); load_thread_params(params, - spl_tables, + spl, test_cfg, - per_table_inserts, + *total_inserts, curr_op, - num_tables, ts, insert_rate, num_insert_threads, @@ -1054,7 +866,7 @@ splinter_perf_inserts(platform_heap_id hid, uint64 total_time = platform_timestamp_elapsed(start_time); timestamp insert_latency_max = 0; uint64 read_io_bytes, write_io_bytes; - cache_io_stats(cc[0], &read_io_bytes, &write_io_bytes); + cache_io_stats(cc, &read_io_bytes, &write_io_bytes); uint64 io_mib = (read_io_bytes + write_io_bytes) / MiB; uint64 bandwidth = (NSEC_TO_SEC(total_time) ? (io_mib / NSEC_TO_SEC(total_time)) : 0); @@ -1085,15 +897,12 @@ splinter_perf_inserts(platform_heap_id hid, NSEC_TO_MSEC(insert_latency_max)); } - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - cache_assert_free(spl->cc); - core_print_insertion_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - core_print_space_use(Platform_default_log_handle, spl); - cache_reset_stats(spl->cc); - // trunk_print(Platform_default_log_handle, spl); - } + cache_assert_free(spl->cc); + core_print_insertion_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + core_print_space_use(Platform_default_log_handle, spl); + cache_reset_stats(spl->cc); + platform_default_log("%s() completed.\n\n", __FUNCTION__); return rc; } @@ -1107,13 +916,11 @@ splinter_perf_inserts(platform_heap_id hid, */ static platform_status splinter_perf_lookups(platform_heap_id hid, - system_config *cfg, test_config *test_cfg, - core_handle *spl_tables, + core_handle *spl, task_system *ts, test_splinter_thread_params *params, uint64 num_lookup_threads, - uint8 num_tables, uint32 max_async_inflight, uint64 total_inserts) { @@ -1125,8 +932,7 @@ splinter_perf_lookups(platform_heap_id hid, uint64 start_time = platform_get_timestamp(); platform_status rc; - do_n_async_ctxt_inits( - hid, num_lookup_threads, num_tables, max_async_inflight, cfg, params); + do_n_async_ctxt_inits(hid, num_lookup_threads, max_async_inflight, params); rc = do_n_thread_creates("lookup_thread", num_lookup_threads, @@ -1156,7 +962,7 @@ splinter_perf_lookups(platform_heap_id hid, timestamp sync_lookup_latency_max = 0; timestamp async_lookup_latency_max = 0; - do_n_async_ctxt_deinits(hid, num_lookup_threads, num_tables, params); + do_n_async_ctxt_deinits(hid, num_lookup_threads, params); for (uint64 i = 0; i < num_lookup_threads; i++) { num_async_lookups += params[i].lookup_stats[ASYNC_LU].num_found @@ -1188,14 +994,12 @@ splinter_perf_lookups(platform_heap_id hid, platform_default_log("max lookup latency ns (sync=%lu, async=%lu)\n", sync_lookup_latency_max, async_lookup_latency_max); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - cache_assert_free(spl->cc); - core_print_insertion_stats(Platform_default_log_handle, spl); - core_print_lookup_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - cache_reset_stats(spl->cc); - } + cache_assert_free(spl->cc); + core_print_insertion_stats(Platform_default_log_handle, spl); + core_print_lookup_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + cache_reset_stats(spl->cc); + platform_default_log("%s() completed.\n\n", __FUNCTION__); return rc; } @@ -1211,13 +1015,11 @@ splinter_perf_lookups(platform_heap_id hid, static platform_status splinter_perf_range_lookups(platform_heap_id hid, test_config *test_cfg, - core_handle *spl_tables, + core_handle *spl, task_system *ts, test_splinter_thread_params *params, - uint64 *per_table_inserts, - uint64 *per_table_ranges, + uint64 num_inserts, uint64 num_range_threads, - uint8 num_tables, trunk_range_perf_params *range_perf) { const char *range_descr = range_perf->range_perf_descr; @@ -1238,24 +1040,15 @@ splinter_perf_range_lookups(platform_heap_id hid, bool32 verbose_progress = test_show_verbose_progress(test_cfg->test_exec_cfg); - uint64 total_ranges = 0; - for (uint8 i = 0; i < num_tables; i++) { - per_table_ranges[i] = - ROUNDUP(per_table_inserts[i] / num_ranges, TEST_RANGE_GRANULARITY); - total_ranges += per_table_ranges[i]; - - if (verbose_progress) { - platform_default_log(" Table[%d], per_table_inserts=%lu" - ", per_table_ranges=%lu, total_ranges=%lu\n", - i, - per_table_inserts[i], - per_table_ranges[i], - total_ranges); - } + uint64 num_ranges_total = + ROUNDUP(num_inserts / num_ranges, TEST_RANGE_GRANULARITY); + if (verbose_progress) { + platform_default_log( + " num_inserts=%lu, num_ranges=%lu\n", num_inserts, num_ranges_total); } for (uint64 i = 0; i < num_range_threads; i++) { - params[i].total_ops = per_table_ranges; + params[i].total_ops = num_ranges_total; params[i].op_granularity = TEST_RANGE_GRANULARITY; params[i].min_range_length = min_range_length; params[i].max_range_length = max_range_length; @@ -1264,10 +1057,10 @@ splinter_perf_range_lookups(platform_heap_id hid, if (verbose_progress) { platform_default_log(" Range thread[%lu] " - "total_ops for table0=%lu, op_granularity=%lu" + "total_ops=%lu, op_granularity=%lu" ", min_range_length=%lu, max_range_length=%lu\n", i, - *params[i].total_ops, + params[i].total_ops, params[i].op_granularity, params[i].min_range_length, params[i].max_range_length); @@ -1341,13 +1134,11 @@ splinter_perf_range_lookups(platform_heap_id hid, (total_time ? SEC_TO_NSEC(num_range_lookups) / total_time : 0), (total_time ? SEC_TO_NSEC(total_returned_tuples) / total_time : 0)); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - cache_assert_free(spl->cc); - core_print_lookup_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - cache_reset_stats(spl->cc); - } + cache_assert_free(spl->cc); + core_print_lookup_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + cache_reset_stats(spl->cc); + platform_default_log("%s() completed.\n\n", __FUNCTION__); return rc; } @@ -1371,36 +1162,27 @@ static platform_status test_splinter_perf(system_config *cfg, test_config *test_cfg, allocator *al, - cache *cc[], + cache *cc, uint64 num_insert_threads, uint64 num_lookup_threads, uint64 num_range_threads, uint32 max_async_inflight, task_system *ts, platform_heap_id hid, - uint8 num_tables, - uint8 num_caches, uint64 insert_rate) { - platform_default_log("splinter_test: SplinterDB performance test started " - "with %d tables\n", - num_tables); - core_handle *spl_tables; + platform_default_log("splinter_test: SplinterDB performance test started\n"); + core_handle *spl; platform_status rc; - rc = test_trunk_create_tables( - &spl_tables, cfg, al, cc, ts, hid, num_tables, num_caches); + rc = test_trunk_create_table(&spl, cfg, al, cc, ts, hid); if (!SUCCESS(rc)) { - platform_error_log("Failed to create splinter table(s): %s\n", + platform_error_log("Failed to create splinter table: %s\n", platform_status_to_string(rc)); return rc; } - uint64 *per_table_inserts = - TYPED_ARRAY_MALLOC(hid, per_table_inserts, num_tables); - uint64 *per_table_ranges = - TYPED_ARRAY_MALLOC(hid, per_table_ranges, num_tables); - uint64 *curr_op = TYPED_ARRAY_ZALLOC(hid, curr_op, num_tables); + uint64 curr_op = 0; uint64 num_threads = MAX(num_insert_threads, num_lookup_threads); num_threads = MAX(num_threads, num_range_threads); @@ -1414,17 +1196,14 @@ test_splinter_perf(system_config *cfg, // if nothing is being inserted. platform_assert(num_insert_threads > 0); rc = splinter_perf_inserts(hid, - cfg, test_cfg, - spl_tables, + spl, cc, ts, params, - per_table_inserts, - curr_op, + &curr_op, num_insert_threads, num_threads, - num_tables, insert_rate, &total_inserts); if (!SUCCESS(rc)) { @@ -1432,15 +1211,13 @@ test_splinter_perf(system_config *cfg, } if (num_lookup_threads > 0) { - ZERO_CONTENTS_N(curr_op, num_tables); - rc = splinter_perf_lookups(hid, - cfg, + curr_op = 0; + rc = splinter_perf_lookups(hid, test_cfg, - spl_tables, + spl, ts, params, num_lookup_threads, - num_tables, max_async_inflight, total_inserts); if (!SUCCESS(rc)) { @@ -1461,16 +1238,14 @@ test_splinter_perf(system_config *cfg, // clang-format on for (int rctr = 0; rctr < ARRAY_SIZE(perf_ranges); rctr++) { - ZERO_CONTENTS_N(curr_op, num_tables); - rc = splinter_perf_range_lookups(hid, + curr_op = 0; + rc = splinter_perf_range_lookups(hid, test_cfg, - spl_tables, + spl, ts, params, - per_table_inserts, - per_table_ranges, + total_inserts, num_range_threads, - num_tables, &perf_ranges[rctr]); if (!SUCCESS(rc)) { goto destroy_splinter; @@ -1479,15 +1254,10 @@ test_splinter_perf(system_config *cfg, } destroy_splinter: - test_trunk_destroy_tables(spl_tables, hid, num_tables); + test_trunk_destroy_table(spl, hid); platform_default_log("After destroy:\n"); - for (uint8 idx = 0; idx < num_caches; idx++) { - cache_print_stats(Platform_default_log_handle, cc[idx]); - } + cache_print_stats(Platform_default_log_handle, cc); platform_free(hid, params); - platform_free(hid, curr_op); - platform_free(hid, per_table_ranges); - platform_free(hid, per_table_inserts); return rc; } @@ -1495,51 +1265,36 @@ platform_status test_splinter_periodic(system_config *cfg, test_config *test_cfg, allocator *al, - cache *cc[], + cache *cc, uint64 num_insert_threads, uint64 num_lookup_threads, uint64 num_range_threads, uint32 max_async_inflight, task_system *ts, platform_heap_id hid, - uint8 num_tables, - uint8 num_caches, uint64 insert_rate) { platform_default_log( - "splinter_test: SplinterDB performance test (periodic) started with " - "%d tables\n", - num_tables); - core_handle *spl_tables; + "splinter_test: SplinterDB performance test (periodic) started\n"); + core_handle *spl; platform_status rc; - rc = test_trunk_create_tables( - &spl_tables, cfg, al, cc, ts, hid, num_tables, num_caches); + rc = test_trunk_create_table(&spl, cfg, al, cc, ts, hid); if (!SUCCESS(rc)) { - platform_error_log("Failed to create splinter table(s): %s\n", + platform_error_log("Failed to create splinter table: %s\n", platform_status_to_string(rc)); return rc; } - uint64 tuple_size, num_inserts; - uint64 *per_table_inserts = - TYPED_ARRAY_MALLOC(hid, per_table_inserts, num_tables); - uint64 *per_table_ranges = - TYPED_ARRAY_MALLOC(hid, per_table_ranges, num_tables); - uint64 *curr_op = TYPED_ARRAY_ZALLOC(hid, curr_op, num_tables); - uint64 total_inserts = 0; - - for (uint8 i = 0; i < num_tables; i++) { - tuple_size = - test_cfg[i].key_size + generator_average_message_size(test_cfg->gen); - num_inserts = test_cfg[i].tree_size / tuple_size; - if (test_cfg[i].key_type == TEST_PERIODIC) { - test_cfg[i].period = num_inserts; - num_inserts *= test_cfg[i].num_periods; - } - per_table_inserts[i] = ROUNDUP(num_inserts, TEST_INSERT_GRANULARITY); - total_inserts += per_table_inserts[i]; + uint64 curr_op = 0; + uint64 tuple_size = + test_cfg->key_size + generator_average_message_size(test_cfg->gen); + uint64 num_inserts = test_cfg->tree_size / tuple_size; + if (test_cfg->key_type == TEST_PERIODIC) { + test_cfg->period = num_inserts; + num_inserts *= test_cfg->num_periods; } + uint64 total_inserts = ROUNDUP(num_inserts, TEST_INSERT_GRANULARITY); uint64 num_threads = num_insert_threads; @@ -1553,17 +1308,15 @@ test_splinter_periodic(system_config *cfg, test_splinter_thread_params *params = TYPED_ARRAY_ZALLOC(hid, params, num_threads); for (uint64 i = 0; i < num_threads; i++) { - params[i].spl = spl_tables; + params[i].spl = spl; params[i].test_cfg = test_cfg; - params[i].total_ops = per_table_inserts; - params[i].curr_op = curr_op; + params[i].total_ops = total_inserts; + params[i].curr_op = &curr_op; params[i].op_granularity = TEST_INSERT_GRANULARITY; params[i].thread_number = i; params[i].expected_found = TRUE; - params[i].num_tables = num_tables; params[i].ts = ts; params[i].insert_rate = insert_rate / num_insert_threads; - params[i].expected_found = TRUE; } uint64 start_time = platform_get_timestamp(); @@ -1584,7 +1337,7 @@ test_splinter_periodic(system_config *cfg, uint64 total_time = platform_timestamp_elapsed(start_time); timestamp insert_latency_max = 0; uint64 read_io_bytes, write_io_bytes; - cache_io_stats(cc[0], &read_io_bytes, &write_io_bytes); + cache_io_stats(cc, &read_io_bytes, &write_io_bytes); uint64 io_mib = (read_io_bytes + write_io_bytes) / MiB; uint64 bandwidth = io_mib / NSEC_TO_SEC(total_time); @@ -1613,16 +1366,13 @@ test_splinter_periodic(system_config *cfg, NSEC_TO_MSEC(insert_latency_max)); } - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - cache_assert_free(spl->cc); - core_print_insertion_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - core_print_space_use(Platform_default_log_handle, spl); - cache_reset_stats(spl->cc); - } + cache_assert_free(spl->cc); + core_print_insertion_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + core_print_space_use(Platform_default_log_handle, spl); + cache_reset_stats(spl->cc); - ZERO_CONTENTS_N(curr_op, num_tables); + curr_op = 0; for (uint64 repeat_round = 0; repeat_round < 10; repeat_round++) { platform_default_log("Beginning repeat round %lu\n", repeat_round); @@ -1649,7 +1399,7 @@ test_splinter_periodic(system_config *cfg, total_time = platform_timestamp_elapsed(start_time); insert_latency_max = 0; - cache_io_stats(cc[0], &read_io_bytes, &write_io_bytes); + cache_io_stats(cc, &read_io_bytes, &write_io_bytes); io_mib = (read_io_bytes + write_io_bytes) / MiB; bandwidth = io_mib / NSEC_TO_SEC(total_time); @@ -1678,16 +1428,13 @@ test_splinter_periodic(system_config *cfg, NSEC_TO_MSEC(insert_latency_max)); } - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - cache_assert_free(spl->cc); - core_print_insertion_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - core_print_space_use(Platform_default_log_handle, spl); - cache_reset_stats(spl->cc); - } + cache_assert_free(spl->cc); + core_print_insertion_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + core_print_space_use(Platform_default_log_handle, spl); + cache_reset_stats(spl->cc); - ZERO_CONTENTS_N(curr_op, num_tables); + curr_op = 0; /* * ********** @@ -1700,10 +1447,7 @@ test_splinter_periodic(system_config *cfg, for (uint64 i = 0; i < num_lookup_threads; i++) { platform_status ret; - for (uint8 j = 0; j < num_tables; j++) { - async_ctxt_init( - hid, max_async_inflight, ¶ms[i].async_lookup[j]); - } + async_ctxt_init(hid, max_async_inflight, ¶ms[i].async_lookup); ret = platform_thread_create(¶ms[i].thread, FALSE, test_trunk_lookup_thread, @@ -1722,10 +1466,8 @@ test_splinter_periodic(system_config *cfg, uint64 num_async_lookups = 0; timestamp sync_lookup_latency_max = 0, async_lookup_latency_max = 0; for (uint64 i = 0; i < num_lookup_threads; i++) { - for (uint8 j = 0; j < num_tables; j++) { - async_ctxt_deinit(hid, params[i].async_lookup[j]); - params[i].async_lookup[j] = NULL; - } + async_ctxt_deinit(hid, params[i].async_lookup); + params[i].async_lookup = NULL; num_async_lookups += params[i].lookup_stats[ASYNC_LU].num_found + params[i].lookup_stats[ASYNC_LU].num_not_found; if (params[i].lookup_stats[SYNC_LU].latency_max @@ -1758,31 +1500,22 @@ test_splinter_periodic(system_config *cfg, platform_default_log("max lookup latency ns (sync=%lu, async=%lu)\n", sync_lookup_latency_max, async_lookup_latency_max); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - cache_assert_free(spl->cc); - core_print_lookup_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - cache_reset_stats(spl->cc); - } + cache_assert_free(spl->cc); + core_print_lookup_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + cache_reset_stats(spl->cc); } - uint64 total_ranges = 0; - - for (uint8 i = 0; i < num_tables; i++) { - per_table_ranges[i] = - ROUNDUP(per_table_inserts[i] / 128, TEST_RANGE_GRANULARITY); - total_ranges += per_table_ranges[i]; - } + uint64 total_ranges = ROUNDUP(total_inserts / 128, TEST_RANGE_GRANULARITY); for (uint64 i = 0; i < num_threads; i++) { - params[i].total_ops = per_table_ranges; + params[i].total_ops = total_ranges; params[i].op_granularity = TEST_RANGE_GRANULARITY; params[i].min_range_length = 1; params[i].max_range_length = 100; } - ZERO_CONTENTS_N(curr_op, num_tables); + curr_op = 0; if (num_range_threads != 0) { start_time = platform_get_timestamp(); @@ -1814,23 +1547,15 @@ test_splinter_periodic(system_config *cfg, total_time * num_range_threads / total_ranges); platform_default_log("splinter total range rate: %lu ops/second\n", SEC_TO_NSEC(total_ranges) / total_time); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - cache_assert_free(spl->cc); - core_print_lookup_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - cache_reset_stats(spl->cc); - } + cache_assert_free(spl->cc); + core_print_lookup_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + cache_reset_stats(spl->cc); - ZERO_CONTENTS_N(curr_op, num_tables); - total_ranges = 0; - for (uint8 i = 0; i < num_tables; i++) { - per_table_ranges[i] = - ROUNDUP(per_table_ranges[i] / 4, TEST_RANGE_GRANULARITY); - total_ranges += per_table_ranges[i]; - } + curr_op = 0; + total_ranges = ROUNDUP(total_ranges / 4, TEST_RANGE_GRANULARITY); for (uint64 i = 0; i < num_range_threads; i++) { - params[i].total_ops = per_table_ranges; + params[i].total_ops = total_ranges; params[i].op_granularity = TEST_RANGE_GRANULARITY; params[i].min_range_length = 512; params[i].max_range_length = 1024; @@ -1865,23 +1590,15 @@ test_splinter_periodic(system_config *cfg, total_time * num_range_threads / total_ranges); platform_default_log("splinter total range rate: %lu ops/second\n", SEC_TO_NSEC(total_ranges) / total_time); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - cache_assert_free(spl->cc); - core_print_lookup_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - cache_reset_stats(spl->cc); - } + cache_assert_free(spl->cc); + core_print_lookup_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + cache_reset_stats(spl->cc); - ZERO_CONTENTS_N(curr_op, num_tables); - total_ranges = 0; - for (uint8 i = 0; i < num_tables; i++) { - per_table_ranges[i] = - ROUNDUP(per_table_ranges[i] / 4, TEST_RANGE_GRANULARITY); - total_ranges += per_table_ranges[i]; - } + curr_op = 0; + total_ranges = ROUNDUP(total_ranges / 4, TEST_RANGE_GRANULARITY); for (uint64 i = 0; i < num_range_threads; i++) { - params[i].total_ops = per_table_ranges; + params[i].total_ops = total_ranges; params[i].op_granularity = TEST_RANGE_GRANULARITY; params[i].min_range_length = 131072 - 16384; params[i].max_range_length = 131072; @@ -1916,25 +1633,17 @@ test_splinter_periodic(system_config *cfg, total_time * num_range_threads / total_ranges); platform_default_log("splinter total range rate: %lu ops/second\n", SEC_TO_NSEC(total_ranges) / total_time); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - cache_assert_free(spl->cc); - core_print_lookup_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - cache_reset_stats(spl->cc); - } + cache_assert_free(spl->cc); + core_print_lookup_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + cache_reset_stats(spl->cc); } destroy_splinter: - test_trunk_destroy_tables(spl_tables, hid, num_tables); + test_trunk_destroy_table(spl, hid); platform_default_log("After destroy:\n"); - for (uint8 idx = 0; idx < num_caches; idx++) { - cache_print_stats(Platform_default_log_handle, cc[idx]); - } + cache_print_stats(Platform_default_log_handle, cc); platform_free(hid, params); - platform_free(hid, curr_op); - platform_free(hid, per_table_ranges); - platform_free(hid, per_table_inserts); return rc; } @@ -1954,7 +1663,7 @@ platform_status test_splinter_parallel_perf(system_config *cfg, test_config *test_cfg, allocator *al, - cache *cc[], + cache *cc, uint64 seed, const uint64 num_threads, const uint64 num_inserts_per_thread, @@ -1962,50 +1671,39 @@ test_splinter_parallel_perf(system_config *cfg, uint32 max_async_inflight, uint8 lookup_positive_pct, task_system *ts, - platform_heap_id hid, - uint8 num_tables, - uint8 num_caches) + platform_heap_id hid) { platform_assert((num_threads > 0), "num_threads=%lu", num_threads); platform_default_log( - "splinter_test: SplinterDB parallel performance test started with " - "%d tables\n", - num_tables); - core_handle *spl_tables; + "splinter_test: SplinterDB parallel performance test started\n"); + core_handle *spl; platform_status rc; platform_assert(num_inserts_per_thread <= num_lookups_per_thread); - rc = test_trunk_create_tables( - &spl_tables, cfg, al, cc, ts, hid, num_tables, num_caches); + rc = test_trunk_create_table(&spl, cfg, al, cc, ts, hid); if (!SUCCESS(rc)) { - platform_error_log("Failed to create splinter table(s): %s\n", + platform_error_log("Failed to create splinter table: %s\n", platform_status_to_string(rc)); return rc; } - uint64 *per_table_inserts = - TYPED_ARRAY_MALLOC(hid, per_table_inserts, num_tables); - uint64 *curr_insert_op = TYPED_ARRAY_ZALLOC(hid, curr_insert_op, num_tables); + uint64 curr_insert_op = 0; // This bit here onwards is very similar to splinter_perf_inserts(), but we // cannot directly call that function as the thread-handler for parallel- // performance exerciser is a different one; see below. - uint64 total_inserts = 0; - - total_inserts = - compute_per_table_inserts(per_table_inserts, cfg, test_cfg, num_tables); + uint64 total_inserts = compute_num_inserts(test_cfg); test_splinter_thread_params *params = TYPED_ARRAY_ZALLOC(hid, params, num_threads); // Load thread-specific exec-parameters for the insert workload load_thread_params(params, - spl_tables, + spl, test_cfg, - per_table_inserts, - curr_insert_op, - num_tables, + total_inserts, + &curr_insert_op, ts, (uint64)0, // unused num_insert_threads (uint64)0, // unused num_threads @@ -2015,14 +1713,13 @@ test_splinter_parallel_perf(system_config *cfg, // Load thread-specific execution parameters for the parallelism between // inserts & lookups in the workload for (uint64 i = 0; i < num_threads; i++) { - params[i].spl = spl_tables; + params[i].spl = spl; params[i].test_cfg = test_cfg; - params[i].total_ops = per_table_inserts; + params[i].total_ops = total_inserts; params[i].op_granularity = TEST_INSERT_GRANULARITY; params[i].thread_number = i; - params[i].num_tables = num_tables; params[i].ts = ts; - params[i].curr_op = curr_insert_op; + params[i].curr_op = &curr_insert_op; params[i].num_ops_per_thread[OP_INSERT] = num_inserts_per_thread; params[i].num_ops_per_thread[OP_LOOKUP] = num_lookups_per_thread; params[i].lookup_positive_pct = lookup_positive_pct; @@ -2031,8 +1728,7 @@ test_splinter_parallel_perf(system_config *cfg, uint64 start_time = platform_get_timestamp(); - do_n_async_ctxt_inits( - hid, num_threads, num_tables, max_async_inflight, cfg, params); + do_n_async_ctxt_inits(hid, num_threads, max_async_inflight, params); rc = do_n_thread_creates("insert/lookup thread", num_threads, @@ -2058,7 +1754,7 @@ test_splinter_parallel_perf(system_config *cfg, timestamp sync_lookup_latency_max = 0, async_lookup_latency_max = 0; uint64 total_insert_duration = 0; - do_n_async_ctxt_deinits(hid, num_threads, num_tables, params); + do_n_async_ctxt_deinits(hid, num_threads, params); for (uint64 i = 0; i < num_threads; i++) { insert_latency_max = @@ -2099,10 +1795,7 @@ test_splinter_parallel_perf(system_config *cfg, NSEC_TO_MSEC(insert_latency_max)); } - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - core_print_insertion_stats(Platform_default_log_handle, spl); - } + core_print_insertion_stats(Platform_default_log_handle, spl); if (num_threads > 0) { uint64 num_async_lookups = @@ -2127,23 +1820,16 @@ test_splinter_parallel_perf(system_config *cfg, platform_default_log("max lookup latency ns (sync=%lu, async=%lu)\n", sync_lookup_latency_max, async_lookup_latency_max); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - core_print_lookup_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - cache_reset_stats(spl->cc); - } + core_print_lookup_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); + cache_reset_stats(spl->cc); } destroy_splinter: - test_trunk_destroy_tables(spl_tables, hid, num_tables); + test_trunk_destroy_table(spl, hid); platform_default_log("After destroy:\n"); - for (uint8 idx = 0; idx < num_caches; idx++) { - cache_print_stats(Platform_default_log_handle, cc[idx]); - } + cache_print_stats(Platform_default_log_handle, cc); platform_free(hid, params); - platform_free(hid, curr_insert_op); - platform_free(hid, per_table_inserts); return rc; } @@ -2151,44 +1837,31 @@ platform_status test_splinter_delete(system_config *cfg, test_config *test_cfg, allocator *al, - cache *cc[], + cache *cc, uint64 num_insert_threads, uint64 num_lookup_threads, uint32 max_async_inflight, task_system *ts, platform_heap_id hid, - uint8 num_tables, - uint8 num_caches, uint64 insert_rate) { - platform_default_log("splinter_test: SplinterDB deletion test started with " - "%d tables\n", - num_tables); - core_handle *spl_tables; + platform_default_log("splinter_test: SplinterDB deletion test started\n"); + core_handle *spl; platform_status rc; - rc = test_trunk_create_tables( - &spl_tables, cfg, al, cc, ts, hid, num_tables, num_caches); + rc = test_trunk_create_table(&spl, cfg, al, cc, ts, hid); if (!SUCCESS(rc)) { - platform_error_log("Failed to initialize splinter table(s): %s\n", + platform_error_log("Failed to initialize splinter table: %s\n", platform_status_to_string(rc)); return rc; } - uint64 tuple_size, num_inserts; - uint64 *per_table_inserts = - TYPED_ARRAY_MALLOC(hid, per_table_inserts, num_tables); - uint64 *curr_op = TYPED_ARRAY_ZALLOC(hid, curr_op, num_tables); - uint64 total_inserts = 0; - - for (uint8 i = 0; i < num_tables; i++) { - tuple_size = - test_cfg[i].key_size + generator_average_message_size(test_cfg->gen); - num_inserts = test_cfg[i].tree_size / tuple_size; - per_table_inserts[i] = ROUNDUP(num_inserts, TEST_INSERT_GRANULARITY); - total_inserts += per_table_inserts[i]; - } - uint64 num_threads = num_insert_threads; + uint64 curr_op = 0; + uint64 tuple_size = + test_cfg->key_size + generator_average_message_size(test_cfg->gen); + uint64 num_inserts = test_cfg->tree_size / tuple_size; + uint64 total_inserts = ROUNDUP(num_inserts, TEST_INSERT_GRANULARITY); + uint64 num_threads = num_insert_threads; if (num_lookup_threads > num_threads) { num_threads = num_lookup_threads; @@ -2199,13 +1872,12 @@ test_splinter_delete(system_config *cfg, ZERO_CONTENTS_N(params, num_threads); for (uint64 i = 0; i < num_threads; i++) { - params[i].spl = spl_tables; + params[i].spl = spl; params[i].test_cfg = test_cfg; - params[i].total_ops = per_table_inserts; - params[i].curr_op = curr_op; + params[i].total_ops = total_inserts; + params[i].curr_op = &curr_op; params[i].op_granularity = TEST_INSERT_GRANULARITY; params[i].thread_number = i; - params[i].num_tables = num_tables; params[i].ts = ts; params[i].insert_rate = insert_rate / num_insert_threads; } @@ -2233,11 +1905,8 @@ test_splinter_delete(system_config *cfg, "splinter total insertion rate: %lu insertions/second\n", SEC_TO_NSEC(total_inserts) / total_time); platform_default_log("After inserts:\n"); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - core_print_insertion_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - } + core_print_insertion_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); for (uint64 i = 0; i < num_insert_threads; i++) { if (!SUCCESS(params[i].rc)) { @@ -2250,7 +1919,7 @@ test_splinter_delete(system_config *cfg, // Deletes message_generate_set_message_type(test_cfg->gen, MESSAGE_TYPE_DELETE); - ZERO_CONTENTS_N(curr_op, num_tables); + curr_op = 0; start_time = platform_get_timestamp(); for (uint64 i = 0; i < num_insert_threads; i++) { platform_status ret; @@ -2270,11 +1939,8 @@ test_splinter_delete(system_config *cfg, platform_default_log("splinter total deletion rate: %lu insertions/second\n", SEC_TO_NSEC(total_inserts) / total_time); platform_default_log("After deletes:\n"); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - core_print_insertion_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - } + core_print_insertion_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); for (uint64 i = 0; i < num_insert_threads; i++) { if (!SUCCESS(params[i].rc)) { @@ -2289,13 +1955,11 @@ test_splinter_delete(system_config *cfg, for (uint64 i = 0; i < num_threads; i++) { params[i].expected_found = FALSE; } - ZERO_CONTENTS_N(curr_op, num_tables); + curr_op = 0; start_time = platform_get_timestamp(); for (uint64 i = 0; i < num_lookup_threads; i++) { - for (uint8 j = 0; j < num_tables; j++) { - async_ctxt_init(hid, max_async_inflight, ¶ms[i].async_lookup[j]); - } + async_ctxt_init(hid, max_async_inflight, ¶ms[i].async_lookup); rc = platform_thread_create( ¶ms[i].thread, FALSE, test_trunk_lookup_thread, ¶ms[i], hid); if (!SUCCESS(rc)) { @@ -2313,10 +1977,8 @@ test_splinter_delete(system_config *cfg, uint64 num_async_lookups = 0; for (uint64 i = 0; i < num_lookup_threads; i++) { - for (uint8 j = 0; j < num_tables; j++) { - async_ctxt_deinit(hid, params[i].async_lookup[j]); - params[i].async_lookup[j] = NULL; - } + async_ctxt_deinit(hid, params[i].async_lookup); + params[i].async_lookup = NULL; num_async_lookups += params[i].lookup_stats[ASYNC_LU].num_found + params[i].lookup_stats[ASYNC_LU].num_not_found; if (!SUCCESS(params[i].rc)) { @@ -2335,21 +1997,14 @@ test_splinter_delete(system_config *cfg, SEC_TO_NSEC(total_inserts) / total_time); platform_default_log("%lu%% lookups were async\n", num_async_lookups * 100 / total_inserts); - for (uint8 spl_idx = 0; spl_idx < num_tables; spl_idx++) { - core_handle *spl = &spl_tables[spl_idx]; - core_print_lookup_stats(Platform_default_log_handle, spl); - cache_print_stats(Platform_default_log_handle, spl->cc); - } + core_print_lookup_stats(Platform_default_log_handle, spl); + cache_print_stats(Platform_default_log_handle, spl->cc); destroy_splinter: - test_trunk_destroy_tables(spl_tables, hid, num_tables); + test_trunk_destroy_table(spl, hid); platform_default_log("After destroy:\n"); - for (uint8 idx = 0; idx < num_caches; idx++) { - cache_print_stats(Platform_default_log_handle, cc[idx]); - } + cache_print_stats(Platform_default_log_handle, cc); platform_free(hid, params); - platform_free(hid, curr_op); - platform_free(hid, per_table_inserts); return rc; } @@ -2371,7 +2026,7 @@ usage(const char *argv0) "--lookup-positive-percent [num] --seed [num]\n" "\t%s --functionality NUM_INSERTS CORRECTNESS_CHECK_FREQUENCY\n" "\t --max-async-inflight [num]\n" - "\t%s --num-tables [ --cache-per-table ] [ --use-shmem ]\n" + "\t%s [ --use-shmem ]\n" "\t%s --insert-rate (inserts_done_by_all_threads in a second)\n", argv0, argv0, @@ -2484,10 +2139,8 @@ splinter_test(int argc, char *argv[]) // Max async IOs inflight per-thread uint32 num_insert_threads, num_lookup_threads; uint32 num_range_lookup_threads, max_async_inflight; - uint32 num_pthreads = 0; - uint8 num_tables = 1; - bool32 cache_per_table = FALSE; - uint64 insert_rate = 0; // no rate throttling by default. + uint32 num_pthreads = 0; + uint64 insert_rate = 0; // no rate throttling by default. task_system ts; bool use_shmem = FALSE; uint8 lookup_positive_pct = 0; @@ -2583,30 +2236,6 @@ splinter_test(int argc, char *argv[]) * IF there are any more arguments remaining, parse them in sequence. * This set of args are expected to come in exactly this order. */ - if (config_argc > 0 - && strncmp(config_argv[0], "--num-tables", sizeof("--num-tables")) == 0) - { - if (config_argc < 2) { - usage(argv[0]); - return -1; - } - if (!try_string_to_uint8(config_argv[1], &num_tables)) { - usage(argv[0]); - return -1; - } - config_argc -= 2; - config_argv += 2; - } - - if (config_argc > 0 - && strncmp( - config_argv[0], "--cache-per-table", sizeof("--cache-per-table")) - == 0) - { - cache_per_table = TRUE; - config_argc -= 1; - config_argv += 1; - } if (config_argc > 0 && strncmp(config_argv[0], "--use-shmem", sizeof("--use-shmem")) == 0) { @@ -2647,14 +2276,6 @@ splinter_test(int argc, char *argv[]) config_argv += 2; } - /* - * Allocate 1 GiB for each cache, or 0.5 GiB for each splinter table, - * whichever is greater. - * Heap capacity should be within [2 * GiB, UINT32_MAX]. - */ - uint8 num_caches = cache_per_table ? num_tables : 1; - // uint64 heap_capacity = MAX(1024 * MiB * num_caches, 512 * MiB * - // num_tables); heap_capacity = MIN(heap_capacity, UINT32_MAX); uint64 heap_capacity = 512 * MiB; if (use_shmem) { platform_default_log( @@ -2672,17 +2293,12 @@ splinter_test(int argc, char *argv[]) * 2. Parse test_config options, see test_config_usage() */ - test_config *test_cfg = TYPED_ARRAY_ZALLOC(hid, test_cfg, num_tables); - for (uint8 i = 0; i < num_tables; i++) { - test_config_set_defaults(test, &test_cfg[i]); - - // All tables share the same message-generator. - test_cfg[i].gen = &gen; + test_config *test_cfg = TYPED_ZALLOC(hid, test_cfg); + test_config_set_defaults(test, test_cfg); + test_cfg->gen = &gen; + test_cfg->test_exec_cfg = &test_exec_cfg; - // And, all tests & tables share the same test-execution parameters. - test_cfg[i].test_exec_cfg = &test_exec_cfg; - } - rc = test_config_parse(test_cfg, num_tables, &config_argc, &config_argv); + rc = test_config_parse(test_cfg, 1, &config_argc, &config_argv); if (!SUCCESS(rc)) { platform_error_log("splinter_test: failed to parse config: %s\n", platform_status_to_string(rc)); @@ -2693,32 +2309,16 @@ splinter_test(int argc, char *argv[]) /* * 3. Parse trunk_config options, see config_usage() */ - system_config *system_cfg = TYPED_ARRAY_MALLOC(hid, system_cfg, num_tables); - test_workload_config *workload_cfg = - TYPED_ARRAY_MALLOC(hid, workload_cfg, num_tables); + system_config *system_cfg = TYPED_ZALLOC(hid, system_cfg); + test_workload_config *workload_cfg = TYPED_ZALLOC(hid, workload_cfg); rc = test_parse_args_n(system_cfg, &test_exec_cfg, workload_cfg, &gen, - num_tables, + 1, config_argc, config_argv); - - // if there are multiple cache capacity, cache_per_table needs to be TRUE - bool32 multi_cap = FALSE; - for (uint8 i = 0; i < num_tables; i++) { - if (system_cfg[i].cache_cfg.capacity != system_cfg[0].cache_cfg.capacity) - { - multi_cap = TRUE; - break; - } - } - if (multi_cap && !cache_per_table) { - platform_error_log("Multiple cache-capacity set but cache-per-table is " - "not set\n"); - rc = STATUS_BAD_PARAM; - } if (!SUCCESS(rc)) { platform_error_log("Failed to parse config arguments. See " "'driver_test %s --help' for usage information: %s\n", @@ -2727,9 +2327,7 @@ splinter_test(int argc, char *argv[]) goto cfg_free; } - for (uint8 i = 0; i < num_tables; i++) { - test_cfg[i].key_size = workload_cfg[i].key_size; - } + test_cfg->key_size = workload_cfg->key_size; seed = test_exec_cfg.seed; @@ -2738,26 +2336,26 @@ splinter_test(int argc, char *argv[]) MAX(num_lookup_threads, MAX(num_insert_threads, num_pthreads)); for (task_type type = 0; type != NUM_TASK_TYPES; type++) { - total_threads += system_cfg[0].task_cfg.num_background_threads[type]; + total_threads += system_cfg->task_cfg.num_background_threads[type]; } // Check if IO subsystem has enough reqs for max async IOs inflight - if (system_cfg[0].io_cfg.kernel_queue_size + if (system_cfg->io_cfg.kernel_queue_size < total_threads * max_async_inflight) { - system_cfg[0].io_cfg.kernel_queue_size = + system_cfg->io_cfg.kernel_queue_size = ROUNDUP(total_threads * max_async_inflight, 32); platform_default_log("Bumped up IO queue size to %lu\n", - system_cfg[0].io_cfg.kernel_queue_size); + system_cfg->io_cfg.kernel_queue_size); } - io_handle *io = io_handle_create(&system_cfg[0].io_cfg, hid); + io_handle *io = io_handle_create(&system_cfg->io_cfg, hid); if (io == NULL) { platform_error_log("Failed to create IO handle\n"); rc = STATUS_NO_MEMORY; goto cfg_free; } - rc = test_init_task_system(&ts, hid, &system_cfg[0].task_cfg); + rc = test_init_task_system(&ts, hid, &system_cfg->task_cfg); if (!SUCCESS(rc)) { platform_error_log("Failed to init splinter state: %s\n", platform_status_to_string(rc)); @@ -2766,44 +2364,34 @@ splinter_test(int argc, char *argv[]) rc_allocator al; rc_allocator_init( - &al, &system_cfg[0].allocator_cfg, io, hid, platform_get_module_id()); + &al, &system_cfg->allocator_cfg, io, hid, platform_get_module_id()); - platform_error_log("Running splinter_test with %d caches\n", num_caches); - clockcache *cc = TYPED_ARRAY_MALLOC(hid, cc, num_caches); + clockcache *cc = TYPED_ZALLOC(hid, cc); platform_assert(cc != NULL); - for (uint8 idx = 0; idx < num_caches; idx++) { - rc = clockcache_init(&cc[idx], - &system_cfg[idx].cache_cfg, - io, - (allocator *)&al, - "test", - hid, - platform_get_module_id()); - platform_assert_status_ok(rc); - } - allocator *alp = (allocator *)&al; + rc = clockcache_init(cc, + &system_cfg->cache_cfg, + io, + (allocator *)&al, + "test", + hid, + platform_get_module_id()); + platform_assert_status_ok(rc); - // Allocate an array of cache pointers to pass around. - cache **caches = TYPED_ARRAY_MALLOC(hid, caches, num_caches); - platform_assert(caches != NULL); - for (uint8 i = 0; i < num_caches; i++) { - caches[i] = (cache *)&cc[i]; - } + allocator *alp = (allocator *)&al; + cache *ccp = (cache *)cc; switch (test) { case perf: rc = test_splinter_perf(system_cfg, test_cfg, alp, - caches, + ccp, num_insert_threads, num_lookup_threads, num_range_lookup_threads, max_async_inflight, &ts, hid, - num_tables, - num_caches, insert_rate); platform_assert(SUCCESS(rc)); break; @@ -2811,14 +2399,12 @@ splinter_test(int argc, char *argv[]) rc = test_splinter_delete(system_cfg, test_cfg, alp, - caches, + ccp, num_insert_threads, num_lookup_threads, max_async_inflight, &ts, hid, - num_tables, - num_caches, insert_rate); platform_assert(SUCCESS(rc)); break; @@ -2826,15 +2412,13 @@ splinter_test(int argc, char *argv[]) rc = test_splinter_perf(system_cfg, test_cfg, alp, - caches, + ccp, num_insert_threads, num_lookup_threads, num_range_lookup_threads, max_async_inflight, &ts, hid, - num_tables, - num_caches, insert_rate); platform_assert(SUCCESS(rc)); break; @@ -2842,29 +2426,27 @@ splinter_test(int argc, char *argv[]) rc = test_splinter_perf(system_cfg, test_cfg, alp, - caches, + ccp, num_insert_threads, num_lookup_threads, num_range_lookup_threads, max_async_inflight, &ts, hid, - num_tables, - num_caches, insert_rate); platform_assert(SUCCESS(rc)); break; case parallel_perf: platform_assert( max_async_inflight == 0 - || (0 < system_cfg[0] - .task_cfg.num_background_threads[TASK_TYPE_MEMTABLE] - && 0 < system_cfg[0] - .task_cfg.num_background_threads[TASK_TYPE_NORMAL])); + || (0 < system_cfg->task_cfg + .num_background_threads[TASK_TYPE_MEMTABLE] + && 0 < system_cfg->task_cfg + .num_background_threads[TASK_TYPE_NORMAL])); rc = test_splinter_parallel_perf(system_cfg, test_cfg, alp, - caches, + ccp, seed, num_pthreads, 3, @@ -2872,35 +2454,29 @@ splinter_test(int argc, char *argv[]) max_async_inflight, lookup_positive_pct, &ts, - hid, - num_tables, - num_caches); + hid); platform_assert_status_ok(rc); break; case periodic: rc = test_splinter_periodic(system_cfg, test_cfg, alp, - caches, + ccp, num_insert_threads, num_lookup_threads, num_range_lookup_threads, max_async_inflight, &ts, hid, - num_tables, - num_caches, insert_rate); platform_assert(SUCCESS(rc)); break; case functionality: - for (uint8 i = 0; i < num_tables; i++) { - system_cfg[i].splinter_cfg.data_cfg->key_to_string = - test_data_config->key_to_string; - } + system_cfg->splinter_cfg.data_cfg->key_to_string = + test_data_config->key_to_string; rc = test_functionality(alp, io, - caches, + ccp, system_cfg, workload_cfg, seed, @@ -2908,8 +2484,6 @@ splinter_test(int argc, char *argv[]) correctness_check_frequency, &ts, hid, - num_tables, - num_caches, max_async_inflight); platform_assert_status_ok(rc); break; @@ -2926,10 +2500,7 @@ splinter_test(int argc, char *argv[]) platform_assert(0); } - for (uint8 idx = 0; idx < num_caches; idx++) { - clockcache_deinit(&cc[idx]); - } - platform_free(hid, caches); + clockcache_deinit(cc); platform_free(hid, cc); allocator_assert_noleaks(alp); rc_allocator_deinit(&al); diff --git a/tests/functional/test_functionality.c b/tests/functional/test_functionality.c index d591e6e7..6fd20665 100644 --- a/tests/functional/test_functionality.c +++ b/tests/functional/test_functionality.c @@ -633,14 +633,12 @@ cmp_ptrs(const void *a, const void *b) * where OP is insert, delete, increment, or decrement. * Verifies the results against the shadow. * - * The test does each of these operations for each of the "num_tables" passed - * in as argument. *----------------------------------------------------------------------------- */ platform_status test_functionality(allocator *al, io_handle *io, - cache *cc[], + cache *cc, system_config *cfg, test_workload_config *workload_cfg, uint64 seed, @@ -648,36 +646,16 @@ test_functionality(allocator *al, uint64 correctness_check_frequency, task_system *state, platform_heap_id hid, - uint8 num_tables, - uint8 num_caches, uint32 max_async_inflight) { - platform_error_log("Functional test started with %d tables\n", num_tables); + platform_error_log("Functional test started\n"); platform_assert(cc != NULL); - /* - * The unified superblock is per-device: core owns one superblock per - * device, so multiple tables sharing one device/allocator is not supported - * until the multi-tree machinery lands (the on-disk format already reserves - * room for it). See the "core owns; 1 table/device" design decision. - */ - platform_assert(num_tables == 1, - "This test currently supports a single table per device " - "(num_tables=%u); rerun with --num-tables 1.", - num_tables); + core_handle *spl = TYPED_ZALLOC(hid, spl); + platform_assert(spl != NULL); - core_handle *spl_tables = TYPED_ARRAY_ZALLOC(hid, spl_tables, num_tables); - platform_assert(spl_tables != NULL); + test_splinter_shadow_tree *shadow = NULL; - test_splinter_shadow_tree **shadows = - TYPED_ARRAY_ZALLOC(hid, shadows, num_tables); - - platform_assert(shadows != NULL); - - allocator_root_id *splinters = - TYPED_ARRAY_ZALLOC(hid, splinters, num_tables); - - platform_assert(splinters != NULL); test_async_lookup *async_lookup; if (max_async_inflight > 0) { async_ctxt_init(hid, max_async_inflight, &async_lookup); @@ -690,50 +668,40 @@ test_functionality(allocator *al, random_init(&prg, seed, 0); - // Initialize the splinter/shadow for each splinter table. - for (uint8 idx = 0; idx < num_tables; idx++) { - cache *cache_to_use = num_caches > 1 ? cc[idx] : *cc; - status = test_splinter_shadow_create(&shadows[idx], hid, num_inserts); - if (!SUCCESS(status)) { - platform_error_log("Failed to init shadow for splinter: %s\n", - platform_status_to_string(status)); - goto cleanup; - } - splinters[idx] = test_generate_allocator_root_id(); - - status = core_mkfs(&spl_tables[idx], - &cfg[idx].splinter_cfg, - al, - cache_to_use, - io, - state, - splinters[idx], - hid); - if (!SUCCESS(status)) { - platform_error_log("core_mkfs() failed for index=%d: %s\n", - idx, - platform_status_to_string(status)); - goto cleanup; - } + // Initialize the splinter table and its shadow. + status = test_splinter_shadow_create(&shadow, hid, num_inserts); + if (!SUCCESS(status)) { + platform_error_log("Failed to init shadow for splinter: %s\n", + platform_status_to_string(status)); + goto cleanup; } - // Validate each tree against an empty shadow. - for (uint8 idx = 0; idx < num_tables; idx++) { - core_handle *spl = &spl_tables[idx]; - test_splinter_shadow_tree *shadow = shadows[idx]; - status = validate_tree_against_shadow(spl, - &prg, - shadow, - hid, - workload_cfg[idx].key_size, - TRUE, - async_lookup); - if (!SUCCESS(status)) { - platform_error_log("Failed to validate empty tree against shadow: \ - %s\n", - platform_status_to_string(status)); - goto cleanup; - } + status = core_mkfs(spl, + &cfg->splinter_cfg, + al, + cc, + io, + state, + test_generate_allocator_root_id(), + hid); + if (!SUCCESS(status)) { + platform_error_log("core_mkfs() failed: %s\n", + platform_status_to_string(status)); + goto cleanup; + } + + // Validate the tree against an empty shadow. + status = validate_tree_against_shadow(spl, + &prg, + shadow, + hid, + workload_cfg->key_size, + TRUE, + async_lookup); + if (!SUCCESS(status)) { + platform_error_log("Failed to validate empty tree against shadow: %s\n", + platform_status_to_string(status)); + goto cleanup; } // Run the test @@ -797,108 +765,68 @@ test_functionality(allocator *al, mindelta, maxdelta); - // Run the main test loop for each table. - for (uint8 idx = 0; idx < num_tables; idx++) { - // cache *cache_to_use = num_caches > 1 ? cc[idx] : *cc; - core_handle *spl = &spl_tables[idx]; - test_splinter_shadow_tree *shadow = shadows[idx]; - // allocator_root_id spl_id = splinters[idx]; - - status = insert_random_messages(spl, - shadow, - &prg, - workload_cfg[idx].key_size, - num_messages, - op, - minkey, - maxkey, - mindelta, - maxdelta); - if (!SUCCESS(status)) { - platform_error_log("Sumpin failed inserting messages: %s\n", - platform_status_to_string(status)); - goto cleanup; - } - - status = validate_tree_against_shadow( - spl, - &prg, - shadow, - hid, - workload_cfg[idx].key_size, - correctness_check_frequency - && (i % correctness_check_frequency) == 0, - async_lookup); - if (!SUCCESS(status)) { - platform_default_log("Failed to validate tree against shadow: %s\n", - platform_status_to_string(status)); - goto cleanup; - } - - /* if (correctness_check_frequency && i != 0 && */ - /* (i % correctness_check_frequency) == 0) { */ - /* platform_assert(trunk_verify_tree(spl)); */ - /* platform_default_log("Dismount and remount\n"); */ - /* allocator_config *al_cfg = ((rc_allocator *)al)->cfg; */ - /* uint64 prev_root_addr = spl->root_addr; */ - /* trunk_dismount(spl); */ - /* rc_allocator_dismount((rc_allocator *)al); */ - /* rc_allocator_mount((rc_allocator *)al, al_cfg, io, hh, hid, */ - /* platform_get_module_id()); */ - /* spl = trunk_mount(&cfg[idx], al, cache_to_use, state, spl_id, - */ - /* hid); */ - /* spl_tables[idx] = spl; */ - /* if (spl->root_addr != prev_root_addr) { */ - /* platform_error_log("Mismatch in root addr across mount\n"); - */ - /* status = STATUS_TEST_FAILED; */ - /* goto cleanup; */ - /* } */ - /* } */ + status = insert_random_messages(spl, + shadow, + &prg, + workload_cfg->key_size, + num_messages, + op, + minkey, + maxkey, + mindelta, + maxdelta); + if (!SUCCESS(status)) { + platform_error_log("Sumpin failed inserting messages: %s\n", + platform_status_to_string(status)); + goto cleanup; } - total_inserts += num_messages; - i++; - } - - // Validate each tree against the shadow one last time. - for (uint8 idx = 0; idx < num_tables; idx++) { - core_handle *spl = &spl_tables[idx]; - test_splinter_shadow_tree *shadow = shadows[idx]; - status = validate_tree_against_shadow( spl, &prg, shadow, hid, - workload_cfg[idx].key_size, - correctness_check_frequency - && ((i - 1) % correctness_check_frequency) != 0, + workload_cfg->key_size, + correctness_check_frequency && (i % correctness_check_frequency) == 0, async_lookup); if (!SUCCESS(status)) { - platform_error_log("Failed to validate tree against shadow one \ - last time: %s\n", - platform_status_to_string(status)); + platform_default_log("Failed to validate tree against shadow: %s\n", + platform_status_to_string(status)); goto cleanup; } + + total_inserts += num_messages; + i++; + } + + // Validate the tree against the shadow one last time. + status = validate_tree_against_shadow( + spl, + &prg, + shadow, + hid, + workload_cfg->key_size, + correctness_check_frequency + && ((i - 1) % correctness_check_frequency) != 0, + async_lookup); + if (!SUCCESS(status)) { + platform_error_log("Failed to validate tree against shadow one last " + "time: %s\n", + platform_status_to_string(status)); + goto cleanup; } cleanup: - for (uint8 idx = 0; idx < num_tables; idx++) { - if (spl_tables[idx].cc != NULL) { - core_destroy(&spl_tables[idx]); - } - if (shadows[idx] != NULL) { - test_splinter_shadow_destroy(hid, shadows[idx]); - } + if (spl->cc != NULL) { + core_destroy(spl); + } + if (shadow != NULL) { + test_splinter_shadow_destroy(hid, shadow); } if (async_lookup) { async_ctxt_deinit(hid, async_lookup); } - platform_free(hid, spl_tables); - platform_free(hid, splinters); - platform_free(hid, shadows); + platform_free(hid, spl); return status; } diff --git a/tests/functional/test_functionality.h b/tests/functional/test_functionality.h index bbbe1a42..ddd51ad5 100644 --- a/tests/functional/test_functionality.h +++ b/tests/functional/test_functionality.h @@ -10,7 +10,7 @@ platform_status test_functionality(allocator *al, io_handle *io, - cache *cc[], + cache *cc, system_config *cfg, test_workload_config *workload_cfg, uint64 seed, @@ -18,6 +18,4 @@ test_functionality(allocator *al, uint64 correctness_check_frequency, task_system *ts, platform_heap_id hid, - uint8 num_tables, - uint8 num_caches, uint32 max_async_inflight); diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index 9be3689d..85c7d28d 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -84,7 +84,6 @@ CTEST_DATA(splinter) uint32 num_insert_threads; uint32 num_lookup_threads; uint32 max_async_inflight; - int spl_num_tables; rc_allocator al; @@ -115,11 +114,9 @@ CTEST_SETUP(splinter) data->num_insert_threads = 1; data->num_lookup_threads = 1; data->max_async_inflight = 64; - data->spl_num_tables = 1; - bool32 cache_per_table = FALSE; - int num_tables = data->spl_num_tables; // Cache, for re-use below - uint8 num_caches = (cache_per_table ? num_tables : 1); + // The config layer still parses per-config arrays; this test uses one. + int num_tables = 1; uint64 heap_capacity = 512 * MiB; // Create a heap for io, allocator, cache and splinter @@ -173,20 +170,18 @@ CTEST_SETUP(splinter) rc_allocator_init(&data->al, &data->system_cfg->allocator_cfg, data->io, data->hid, platform_get_module_id()); - data->clock_cache = TYPED_ARRAY_MALLOC(data->hid, data->clock_cache, num_caches); + data->clock_cache = TYPED_MALLOC(data->hid, data->clock_cache); ASSERT_TRUE((data->clock_cache != NULL)); - for (uint8 idx = 0; idx < num_caches; idx++) { - rc = clockcache_init(&data->clock_cache[idx], - &data->system_cfg[idx].cache_cfg, - data->io, - (allocator *)&data->al, - "test", - data->hid, - platform_get_module_id()); + rc = clockcache_init(data->clock_cache, + &data->system_cfg->cache_cfg, + data->io, + (allocator *)&data->al, + "test", + data->hid, + platform_get_module_id()); - ASSERT_TRUE(SUCCESS(rc), "clockcache_init() failed for index=%d. ", idx); - } + ASSERT_TRUE(SUCCESS(rc), "clockcache_init() failed. "); } // clang-format on From d5d555d2c75b054e13fd9adc37c22a3a1ff67b52 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Thu, 30 Jul 2026 12:44:19 -0700 Subject: [PATCH 62/64] formatting Signed-off-by: Rob Johnson --- tests/functional/splinter_test.c | 22 +++++++++++----------- tests/functional/test_functionality.c | 9 ++------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/tests/functional/splinter_test.c b/tests/functional/splinter_test.c index 9cb88732..6671adb3 100644 --- a/tests/functional/splinter_test.c +++ b/tests/functional/splinter_test.c @@ -144,9 +144,9 @@ test_trunk_insert_thread(void *arg) test_cfg->period); generate_test_message(test_cfg->gen, insert_num, &msg); platform_status rc = core_insert(spl, - key_buffer_key(&keybuf), - merge_accumulator_to_message(&msg), - NULL); + key_buffer_key(&keybuf), + merge_accumulator_to_message(&msg), + NULL); platform_assert_status_ok(rc); if (spl->cfg.use_stats) { ts = platform_timestamp_elapsed(ts); @@ -425,12 +425,12 @@ advance_base(test_splinter_thread_params *params, } else { // negative lookup by select random base // [local_curr_op + op_granularity, total_ops - op_granularity] - random_base = ((random_next_uint64(rs) - % ((total_ops - local_curr_op) / op_granularity - - 1)) - + 1) - * op_granularity - + local_curr_op; + random_base = + ((random_next_uint64(rs) + % ((total_ops - local_curr_op) / op_granularity - 1)) + + 1) + * op_granularity + + local_curr_op; } } *base = random_base; @@ -567,7 +567,7 @@ test_trunk_insert_lookup_thread(void *arg) uint64 op_granularity = params->op_granularity; uint64 seed = params->seed; verify_tuple_arg vtarg = {.stats_only = TRUE, - .stats = ¶ms->lookup_stats[ASYNC_LU]}; + .stats = ¶ms->lookup_stats[ASYNC_LU]}; uint64 bases[NUM_OP_TYPES]; uint64 granularities[NUM_OP_TYPES]; @@ -1919,7 +1919,7 @@ test_splinter_delete(system_config *cfg, // Deletes message_generate_set_message_type(test_cfg->gen, MESSAGE_TYPE_DELETE); - curr_op = 0; + curr_op = 0; start_time = platform_get_timestamp(); for (uint64 i = 0; i < num_insert_threads; i++) { platform_status ret; diff --git a/tests/functional/test_functionality.c b/tests/functional/test_functionality.c index 6fd20665..f1cc61a2 100644 --- a/tests/functional/test_functionality.c +++ b/tests/functional/test_functionality.c @@ -691,13 +691,8 @@ test_functionality(allocator *al, } // Validate the tree against an empty shadow. - status = validate_tree_against_shadow(spl, - &prg, - shadow, - hid, - workload_cfg->key_size, - TRUE, - async_lookup); + status = validate_tree_against_shadow( + spl, &prg, shadow, hid, workload_cfg->key_size, TRUE, async_lookup); if (!SUCCESS(status)) { platform_error_log("Failed to validate empty tree against shadow: %s\n", platform_status_to_string(status)); From 4b12f3fcfbc7b98b8f476d1e1b08b6eb174f2717 Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Thu, 30 Jul 2026 13:10:00 -0700 Subject: [PATCH 63/64] increase shmem size in splinter_test Signed-off-by: Rob Johnson --- tests/unit/splinter_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/splinter_test.c b/tests/unit/splinter_test.c index 85c7d28d..a3301f89 100644 --- a/tests/unit/splinter_test.c +++ b/tests/unit/splinter_test.c @@ -117,7 +117,7 @@ CTEST_SETUP(splinter) // The config layer still parses per-config arrays; this test uses one. int num_tables = 1; - uint64 heap_capacity = 512 * MiB; + uint64 heap_capacity = 1024 * MiB; // Create a heap for io, allocator, cache and splinter platform_status rc = platform_heap_create(platform_get_module_id(), From f83bc0eb2c52a9f5ceb35c28e85fa848c924cb3a Mon Sep 17 00:00:00 2001 From: Rob Johnson Date: Thu, 30 Jul 2026 22:05:04 -0700 Subject: [PATCH 64/64] fix gc bug Signed-off-by: Rob Johnson --- src/trunk.c | 26 +++------ test.sh | 81 +++++++++++++++++++++++++++ tests/functional/test_functionality.c | 1 + 3 files changed, 90 insertions(+), 18 deletions(-) diff --git a/src/trunk.c b/src/trunk.c index e5b7c9ab..aa1eb730 100644 --- a/src/trunk.c +++ b/src/trunk.c @@ -2580,25 +2580,15 @@ trunk_snapshot_release(trunk_context *context, trunk_snapshot *snapshot) } /* - * Route the release through a scratch context so the ordinary COW - * teardown path (trunk_context_deinit -> trunk_ondisk_node_ref_destroy) - * performs the single decrement snapshot's reference is owed. The scratch - * context only needs context's shared cfg/cc/al/ts; it does not need to - * share context's root. + * Perform the single decrement the snapshot's reference is owed against + * `context` itself. It must not be routed through a temporary context: if + * the root is still cache_in_use (a concurrent reader holds it), the + * decrement defers the node's destruction onto context->pending_gcs, and + * only a context that outlives this call will ever drain that list. */ - trunk_context scratch; - platform_status rc = trunk_context_init(&scratch, - context->cfg, - context->hid, - context->cc, - context->al, - context->ts, - *snapshot); - snapshot->root_addr = 0; // trunk_context_init consumes it regardless of rc - if (!SUCCESS(rc)) { - return rc; - } - trunk_context_deinit(&scratch); + uint64 root_addr = snapshot->root_addr; + snapshot->root_addr = 0; + trunk_ondisk_node_dec_ref(context, root_addr); return STATUS_OK; } diff --git a/test.sh b/test.sh index 2ad9ad23..fa9fe962 100755 --- a/test.sh +++ b/test.sh @@ -12,6 +12,52 @@ function run() rm -f db } +# Run a workload that is expected to complete at least one checkpoint, and fail +# if it completes none. +# +# A plain run() would pass whether or not checkpointing ever fired, so a +# regression that quietly stopped taking checkpoints would look green. The +# caller must pass --log (to enable the log, and hence auto-checkpointing) and +# --stats (so core_print_insertion_stats() emits the "| checkpoints:" line this +# parses). Several drivers dump stats more than once and the counter is +# cumulative, so take the largest value printed. +function run_checkpointed() +{ + local logfile + logfile=$(mktemp) + + set +e + run_with_timing "$*" ${BINDIR}/$@ $DB_CAPACITY > "$logfile" 2>&1 + local rc=$? + set -e + + cat "$logfile" + rm -f db + + if [ "$rc" -ne 0 ]; then + rm -f "$logfile" + echo "FAILED: $* exited with status $rc" + exit 1 + fi + + local checkpoints + checkpoints=$(grep -oE '\| checkpoints: +[0-9]+' "$logfile" \ + | grep -oE '[0-9]+' | sort -rn | head -1) + rm -f "$logfile" + + if [ -z "$checkpoints" ]; then + echo "FAILED: $* printed no checkpoint statistics" \ + "(both --log and --stats are required)" + exit 1 + fi + if [ "$checkpoints" -eq 0 ]; then + echo "FAILED: $* completed 0 checkpoints;" \ + "lower --checkpoint-log-size-mib or grow the workload" + exit 1 + fi + echo "PASSED: $* completed ${checkpoints} checkpoints" +} + # 14 minutes function cache_tests_1() { # 25 sec each @@ -52,6 +98,40 @@ function functionality_tests() { run driver_test splinter_test --functionality 10000000 1000 --cache-capacity-mib 4096 } +# 8 minutes +# +# Logging (and hence auto-checkpointing) is off by default in every other group +# here, so without these the checkpoint machinery goes essentially unexercised +# outside the unit tests. --checkpoint-log-size-mib is deliberately far below +# the production default (one cache's worth) so that checkpoints fire many times +# during a short run; run_checkpointed() asserts they actually did. +function checkpoint_tests() { + # 45 sec each. Correctness under checkpointing: the shadow-verified + # workload, which will catch a checkpoint that corrupts or loses data. + run_checkpointed driver_test splinter_test --functionality 500000 100 --seed 135 --log --checkpoint-log-size-mib 2 --stats + run_checkpointed driver_test splinter_test --functionality 500000 100 --use-shmem --seed 135 --log --checkpoint-log-size-mib 2 --stats + + # 60 sec. Concurrent inserts/lookups/range-lookups across a log cut. + run_checkpointed driver_test splinter_test --perf --max-async-inflight 0 --num-insert-threads 4 --num-lookup-threads 4 --num-range-lookup-threads 4 --num-inserts 300000 --cache-capacity-mib 512 --log --checkpoint-log-size-mib 2 --stats --num-normal-bg-threads 2 --num-memtable-bg-threads 2 + + # 90 sec each. Regression coverage for the pending_gcs crash: a checkpoint + # retiring the old root while an async lookup still pins it in the cache + # deferred the node's destruction onto a scratch context that was freed + # immediately after. Needs async lookups AND frequent checkpoints together. + run_checkpointed driver_test splinter_test --parallel-perf --max-async-inflight 10 --num-pthreads 8 --tree-size-mib 512 --num-normal-bg-threads 4 --num-memtable-bg-threads 2 --log --checkpoint-log-size-mib 2 --stats + run_checkpointed driver_test splinter_test --parallel-perf --max-async-inflight 0 --num-pthreads 8 --lookup-positive-percent 10 --tree-size-mib 512 --log --checkpoint-log-size-mib 2 --stats + + # 45 sec. Deletes and repeated overwrite rounds; the overwrite workload is + # the one that never advances the memtable generation, so it exercises the + # log-size trigger rather than the generation-count one. + run_checkpointed driver_test splinter_test --delete --tree-size-mib 512 --log --checkpoint-log-size-mib 2 --stats + run_checkpointed driver_test splinter_test --periodic --tree-size-mib 256 --log --checkpoint-log-size-mib 2 --stats + + # 60 sec. A realistic cadence rather than a pathological one, so the + # coverage does not depend solely on a tiny threshold. + run_checkpointed driver_test splinter_test --parallel-perf --max-async-inflight 10 --num-pthreads 8 --tree-size-mib 512 --num-normal-bg-threads 4 --num-memtable-bg-threads 2 --log --checkpoint-log-size-mib 64 --stats +} + function parallel_perf_test_1() { # 115 sec each run driver_test splinter_test --parallel-perf --max-async-inflight 0 --num-pthreads 8 --lookup-positive-percent 10 --tree-size-gib 8 @@ -174,6 +254,7 @@ function all_tests() { cache_tests_2 cache_tests_3 functionality_tests + checkpoint_tests parallel_perf_test_1 parallel_perf_test_2 parallel_perf_test_3 diff --git a/tests/functional/test_functionality.c b/tests/functional/test_functionality.c index f1cc61a2..56f4ea37 100644 --- a/tests/functional/test_functionality.c +++ b/tests/functional/test_functionality.c @@ -813,6 +813,7 @@ test_functionality(allocator *al, cleanup: if (spl->cc != NULL) { + core_print_insertion_stats(Platform_default_log_handle, spl); core_destroy(spl); } if (shadow != NULL) {