diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index fc6d327817..c10b54f4e3 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -1371,6 +1371,10 @@ dump_grd_recovery(ReturnSetInfo *rsinfo) fmt_int64((int64)c.join_block_views_rebuilt)); emit_row(rsinfo, "grd_recovery", "join_block_recovering_failclosed", fmt_int64((int64)c.join_block_recovering_failclosed)); + /* Shape A (crash-rejoin re-declare barrier): off-path crash-rejoin fence-arm + * events (standalone counter, not part of the snapshot struct). */ + emit_row(rsinfo, "grd_recovery", "offpath_crash_rejoin_fenced", + fmt_int64((int64)cluster_grd_offpath_crash_rejoin_fenced_count())); } /* diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 631736d0aa..1f31eae132 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -1275,9 +1275,43 @@ cluster_gcs_block_fallback_verify_refresh(BufferDesc *buf, BufferTag tag, SCN ex GcsLostWriteVerdict verdict; bool refreshed = false; /* S3 forensics — storage re-read happened */ - if (buf == NULL || !SCN_VALID(expected_scn)) + if (buf == NULL) return; + /* + * fix 2 (crash-rejoin re-declare barrier, defense in depth): an InvalidScn + * master watermark normally SKIPs (not SCN-tracked / old-binary master / + * holder re-ack). But if THIS self-home block is under the off-path crash- + * rejoin fence, the local GRD watermark was wiped by the restart, so an + * Invalid watermark can mask a stale home block — fail-closed instead of + * SKIP, except for a genuine extension block (never cross-node written). + * This is a second line behind the phase-gate boot barrier, which already + * fences the self-home block before the acquire reaches here. + */ + { + bool self_fenced + = (!cluster_online_join && cluster_gcs_lookup_master_static(tag) == cluster_node_id + && cluster_conf_node_count() > 1 && !cluster_grd_offpath_boot_decided()); + ClusterColdGrdVerdict cv = cluster_gcs_cold_grd_watermark_verdict( + SCN_VALID(expected_scn), self_fenced, + self_fenced && cluster_bufmgr_block_is_extension_for_gcs(tag)); + + if (cv == CLUSTER_COLD_GRD_SKIP) + return; + if (cv == CLUSTER_COLD_GRD_FAIL_CLOSED) { + pg_atomic_fetch_add_u64(&ClusterGcsBlock->fallback_scn_failclosed_count, 1); + ereport(ERROR, + (errcode(ERRCODE_CLUSTER_GCS_BLOCK_RESOURCE_RECOVERING), + errmsg("crash-rejoin: cannot prove home block ownership after restart " + "(cold GRD watermark) for tag spc=%u db=%u rel=%u block=%u", + tag.spcOid, tag.dbOid, tag.relNumber, tag.blockNum), + errhint("The block resource is recovering after an unclean restart; retry the " + "transaction, or enable cluster.online_join for an online re-declare " + "rejoin."))); + } + /* CLUSTER_COLD_GRD_PROVE: expected_scn valid — run the normal verdict. */ + } + page_scn = cluster_bufmgr_read_block_scn_for_gcs(buf); verdict = gcs_block_lost_write_verdict(expected_scn, page_scn); if (verdict == GCS_LOST_WRITE_PASS) { @@ -1450,6 +1484,28 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) */ static_master = cluster_gcs_lookup_master_static(tag); + /* + * TT lane / crash-rejoin re-declare barrier (Shape A) — off-path boot + * barrier. With cluster.online_join=off a node that boots into a running + * cluster self-admits immediately (cluster_reconfig.c:206) with an EMPTY + * GRD and NO re-declare episode: for a block whose STATIC home is self, + * the acquire path would find master==self, read the empty local GRD, and + * cold-grant from the stale/empty disk page — a silent stale READ and a + * silently-diverging WRITE (the P0). Until the off-path rejoin tick has + * classified this incarnation (crash-rejoin -> self-fence armed; + * bootstrap -> nothing), self cannot prove its home blocks' ownership, so + * fence them RECOVERING. Both reads and writes reach this gate via + * cluster_pcm_lock_acquire_buffer, so this closes the boot-to-decision + * race with ZERO cold-serve window (Rule 8.A: uncertain -> fail-closed). + * Skipped for online_join=on (its admission + join fence govern) and for + * a single declared node (no peer can hold a conflicting copy). + */ + if (!cluster_online_join && static_master == cluster_node_id && cluster_conf_node_count() > 1 + && !cluster_grd_offpath_boot_decided()) { + cluster_grd_inc_join_block_failclosed(); + return GCS_BLOCK_RECOVERING; + } + /* * spec-5.16 D3 (r1 P1-C) — online-join PCM block snap-back fence, placed * BEFORE the non-DEAD-static-master early NORMAL below. When a joiner (a diff --git a/src/backend/cluster/cluster_grd.c b/src/backend/cluster/cluster_grd.c index 660ff6ae38..f84755a8f8 100644 --- a/src/backend/cluster/cluster_grd.c +++ b/src/backend/cluster/cluster_grd.c @@ -763,11 +763,14 @@ cluster_grd_shmem_init(void) for (i = 0; i < CLUSTER_MAX_NODES; i++) pg_atomic_init_u64(&cluster_grd_state->join_pcm_fence_member_epoch[i], 0); pg_atomic_init_u32(&cluster_grd_state->recovery_direction, (uint32)GRD_REMASTER_DIR_NONE); + /* Shape A: off-path boot barrier starts UNDECIDED (fail-closed). */ + pg_atomic_init_u32(&cluster_grd_state->offpath_boot_decided, 0); pg_atomic_init_u64(&cluster_grd_state->join_remaster_started_count, 0); pg_atomic_init_u64(&cluster_grd_state->join_remaster_done_count, 0); pg_atomic_init_u64(&cluster_grd_state->join_shards_remastered_count, 0); pg_atomic_init_u64(&cluster_grd_state->join_block_views_rebuilt_count, 0); pg_atomic_init_u64(&cluster_grd_state->join_block_recovering_failclosed_count, 0); + pg_atomic_init_u64(&cluster_grd_state->offpath_crash_rejoin_fenced_count, 0); } /* spec-2.15 v0.4 P1.1: entry HTAB allocation gated on GUC. GUC=0 @@ -1799,6 +1802,53 @@ cluster_grd_inc_join_block_failclosed(void) pg_atomic_fetch_add_u64(&cluster_grd_state->join_block_recovering_failclosed_count, 1); } +/* + * Shape A (crash-rejoin re-declare barrier) — off-path boot-barrier flag. + * + * cluster_grd_offpath_boot_decided() -- false until the off-path rejoin + * tick has classified this incarnation (bootstrap vs crash-rejoin). The + * phase gate fences self-home blocks RECOVERING while false, so a node + * that self-admits at boot with cluster.online_join=off cannot cold-serve + * its home blocks before it has proven their ownership (Rule 8.A). + * Defaults DECIDED (true) when the GRD region is absent so a cluster-off + * build never fences. + */ +bool +cluster_grd_offpath_boot_decided(void) +{ + if (cluster_grd_state == NULL) + return true; + return pg_atomic_read_u32(&cluster_grd_state->offpath_boot_decided) != 0; +} + +/* Mark the off-path boot decision complete (idempotent; single writer = the + * reconfig LMON tick). After this the boot barrier lifts; on a crash-rejoin + * the caller has already armed the self-fence, which keeps home blocks + * RECOVERING via the existing join fence check. */ +void +cluster_grd_set_offpath_boot_decided(void) +{ + if (cluster_grd_state != NULL) + pg_atomic_write_u32(&cluster_grd_state->offpath_boot_decided, 1); +} + +/* Shape A observability: count an off-path crash-rejoin fence-arm (LMON single + * writer; read for dump_grd + t/404). */ +void +cluster_grd_inc_offpath_crash_rejoin_fenced(void) +{ + if (cluster_grd_state != NULL) + pg_atomic_fetch_add_u64(&cluster_grd_state->offpath_crash_rejoin_fenced_count, 1); +} + +uint64 +cluster_grd_offpath_crash_rejoin_fenced_count(void) +{ + if (cluster_grd_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->offpath_crash_rejoin_fenced_count); +} + /* spec-4.6 D5 — bulk counter snapshot for the dump path. */ void cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out) diff --git a/src/backend/cluster/cluster_qvotec.c b/src/backend/cluster/cluster_qvotec.c index 916a629399..69b01b83be 100644 --- a/src/backend/cluster/cluster_qvotec.c +++ b/src/backend/cluster/cluster_qvotec.c @@ -140,11 +140,34 @@ typedef struct ClusterQvotecShmem { pg_atomic_uint32 poll_cycle_count; pg_atomic_uint32 torn_write_detect_count; pg_atomic_uint32 _pad; - uint8 _reserved[64]; + /* + * Merge-order reservation (守门 07-15): the convert-queue lane claims + * offset 64..71 for its self_incarnation (pg_atomic_uint64, commit + * ee536b5bb7, StaticAssert-pinned). Queue merges first; this lane rebases + * after and drops this placeholder so self_incarnation occupies 64..71 and + * prior_unclean_death stays at 72. Keeping the byte layout identical now + * makes that rebase a no-op on the wire/shmem image. + */ + uint8 _reserved_queue_self_incarnation[8]; /* offset 64..71 */ + /* + * Crash-rejoin re-declare barrier (Shape A) — set ONCE at qvotec startup + * (before the READY publish), read-only thereafter: 1 iff this node's + * prior-incarnation self-slot on the voting disk still had the ALIVE flag + * set (a clean shutdown clears it via qvotec_clear_self_alive_on_clean_ + * shutdown; a crash / immediate stop does NOT), i.e. this boot follows an + * UNCLEAN death. The off-path rejoin tick fences self-home blocks + + * closes the write gate on this, so a crash-rejoined node never cold- + * serves stale ownership even when it restarts faster than the survivor's + * dead-deadband (the epoch signal is INITIAL on both sides in that race). + */ + pg_atomic_uint32 prior_unclean_death; /* offset 72..75 */ + uint8 _reserved[52]; } ClusterQvotecShmem; StaticAssertDecl(sizeof(ClusterQvotecShmem) == 128, "ClusterQvotecShmem must be exactly 128 bytes (2 cache lines)"); +StaticAssertDecl(offsetof(ClusterQvotecShmem, prior_unclean_death) == 72, + "prior_unclean_death must sit at offset 72 (queue lane owns 64..71)"); static ClusterQvotecShmem *QvotecShmem = NULL; @@ -267,6 +290,9 @@ cluster_qvotec_shmem_init(void) pg_atomic_init_u32(&QvotecShmem->poll_cycle_count, 0); pg_atomic_init_u32(&QvotecShmem->torn_write_detect_count, 0); pg_atomic_init_u32(&QvotecShmem->_pad, 0); + memset(QvotecShmem->_reserved_queue_self_incarnation, 0, + sizeof(QvotecShmem->_reserved_queue_self_incarnation)); + pg_atomic_init_u32(&QvotecShmem->prior_unclean_death, 0); memset(QvotecShmem->_reserved, 0, sizeof(QvotecShmem->_reserved)); } } @@ -372,6 +398,23 @@ cluster_qvotec_get_disks_total_count(void) return (int)pg_atomic_read_u32(&QvotecShmem->disks_total_count); } +/* + * cluster_qvotec_prior_unclean_death -- crash-rejoin re-declare barrier + * (Shape A). True iff this node's prior-incarnation self-slot on the voting + * disk still carried the ALIVE flag at startup (an unclean death: a crash / + * immediate stop that skipped the clean-shutdown ALIVE blank). Latched once + * before the READY publish; stable for the incarnation. False when qvotec is + * absent (no voting disks) so a diskless / single-node deployment is never + * fenced by this signal. + */ +bool +cluster_qvotec_prior_unclean_death(void) +{ + if (QvotecShmem == NULL) + return false; + return pg_atomic_read_u32(&QvotecShmem->prior_unclean_death) != 0; +} + uint64 cluster_qvotec_get_current_epoch_at_boot(void) { @@ -1749,7 +1792,7 @@ ClusterQvotecMain(void) bool ghost_fresh = false; int d; - for (d = 0; d < qvotec_n_disks && !ghost_fresh; d++) { + for (d = 0; d < qvotec_n_disks; d++) { ClusterVotingSlot probe; ClusterVotingDiskIoState rrc; @@ -1759,14 +1802,29 @@ ClusterQvotecMain(void) if (probe.generation == 0) continue; /* never written */ if (!(probe.flags & CLUSTER_VOTING_SLOT_FLAG_ALIVE)) - continue; /* prior shutdown cleared ALIVE — ok */ + continue; /* prior shutdown cleared ALIVE — clean death, ok */ if (probe.incarnation == qvotec_self_incarnation) continue; /* same incarnation — impossible but defensive */ + + /* + * Crash-rejoin re-declare barrier (Shape A) — a prior-incarnation + * self-slot that still carries ALIVE means the previous postmaster + * of THIS node died WITHOUT running the clean-shutdown blank + * (qvotec_clear_self_alive_on_clean_shutdown), i.e. an UNCLEAN + * death. Latch it REGARDLESS of freshness: a stale ALIVE ghost is + * still proof we crashed (we just crashed longer ago), and the + * fence must engage on a fast rejoin where the survivor has not yet + * advanced its epoch (the epoch signal is INITIAL on both sides). + * Single writer, before the READY publish; read-only afterwards. + */ + if (QvotecShmem != NULL) + pg_atomic_write_u32(&QvotecShmem->prior_unclean_death, 1); + if (probe.heartbeat_ts_us == 0) continue; if (now_us > probe.heartbeat_ts_us && (now_us - probe.heartbeat_ts_us) > heartbeat_timeout_us) - continue; /* already stale */ + continue; /* already stale — no fast-restart Q6 sleep needed */ ghost_fresh = true; } diff --git a/src/backend/cluster/cluster_reconfig.c b/src/backend/cluster/cluster_reconfig.c index ccaf5991c1..af6a3d7520 100644 --- a/src/backend/cluster/cluster_reconfig.c +++ b/src/backend/cluster/cluster_reconfig.c @@ -2136,6 +2136,128 @@ cluster_reconfig_joiner_self_tick(void) } +/* + * cluster_reconfig_offpath_rejoin_tick -- crash-rejoin re-declare barrier + * (Shape A), the cluster.online_join=off counterpart of the joiner self-tick. + * + * With online_join=off the joiner self-tick above early-returns, so a node + * that crash-restarts into a running cluster keeps its shmem-init + * self_join_admitted = 1 (cluster_reconfig.c:206) and boots straight to a + * writable MEMBER with an EMPTY GRD and no re-declare episode. For a block + * whose static home is self, the acquire path then cold-grants from the + * stale/empty disk page (silent stale read / silently-diverging write — the + * P0). The phase-gate boot barrier (cluster_gcs_block.c) fences self-home + * blocks RECOVERING from process start (the flag defaults 0) until THIS tick + * classifies the incarnation, so there is zero cold-serve window. + * + * Decision (LMON single-writer; the barrier flag and self_join_admitted are + * both flipped here): + * - single declared node -> decided, no fence (no peer can conflict) + * - crash-rejoin (already run) -> arm the self-fence, THEN demote + * self_join_admitted to 0 (Rule 8.A: never + * raised to 1 before the fence is armed); + * home blocks stay RECOVERING via the join + * fence, writes fail-closed 53R60. The + * survivor re-declare self-heal (fence + * lift) is a separate spec (Shape B). + * - cold bootstrap at INITIAL -> decided, no fence (fresh cluster, no + * stale home blocks) + * - undecided -> leave the barrier up (fail-closed), + * retry next tick + * + * online_join=on takes its own joiner_self_tick / note_self_admitted path + * and never enters here. + */ +static void +cluster_reconfig_offpath_rejoin_tick(void) +{ + static bool offpath_decided_local = false; + + if (ReconfigShmem == NULL || cluster_online_join) + return; /* off path only */ + if (cluster_node_id < 0 || cluster_node_id >= CLUSTER_MAX_NODES) + return; + if (offpath_decided_local) + return; /* once per incarnation (LMON-local) */ + if (cluster_reconfig_is_removed_unlocked(cluster_node_id)) + return; /* a removed node keeps its 53R64 self-demote gate */ + + if (cluster_conf_node_count() <= 1) { + /* Lone declared node: no peer can hold a conflicting copy, so there is + * nothing to re-declare — decide at once so the boot barrier never + * fences a single-node deployment. */ + cluster_grd_set_offpath_boot_decided(); + offpath_decided_local = true; + return; + } + + /* + * REJOIN when EITHER signal fires: + * already_running -- a declared peer is observed past INITIAL (the + * survivor already reconfigured; slow rejoin). + * prior_unclean_death -- this node's prior-incarnation voting-disk + * self-slot still carried ALIVE (an unclean + * death). This is the ONLY signal that fires on + * a FAST rejoin, where the node restarts inside + * the survivor's dead-deadband so BOTH sides are + * still at epoch INITIAL and the epoch signal is + * blind (守门裁决 07-15: ALIVE bit, not epoch). + * The clean-shutdown blank clears ALIVE (keeps epoch), so a genuine clean + * co-boot never trips prior_unclean_death. + */ + if (cluster_reconfig_cluster_already_running() || cluster_qvotec_prior_unclean_death()) { + uint8 self_set[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + + /* + * Arm the epoch-keyed join fence too (belt: it engages on a SLOW + * rejoin where this node adopted a peer epoch > INITIAL). On a FAST + * rejoin this node is still at CLUSTER_EPOCH_INITIAL, so the monotonic- + * max fence epoch cannot rise above 0 and the join fence is a no-op — + * the READ fence is therefore carried by the boot barrier below, which + * is epoch-independent: we deliberately DO NOT set offpath_boot_decided, + * so the phase gate keeps self-home blocks RECOVERING for the whole + * incarnation (fail-closed until a clean restart / online_join). + */ + self_set[cluster_node_id >> 3] = (uint8)(1u << (cluster_node_id & 7)); + cluster_grd_arm_join_pcm_fence(self_set); /* fence FIRST (8.A) */ + + LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); + ReconfigShmem->self_join_admitted = 0; /* then close the write gate */ + LWLockRelease(&ReconfigShmem->lock); + + /* NB: offpath_boot_decided stays 0 -> the boot barrier persists as the + * read fence for this incarnation. offpath_decided_local latches the + * tick so it does not re-arm / re-log every cycle. */ + cluster_grd_inc_offpath_crash_rejoin_fenced(); + offpath_decided_local = true; + + ereport(LOG, + (errmsg("cluster membership: node %d crash-rejoin detected (cluster.online_join=" + "off%s) — home blocks fenced and writes closed (53R60) to avoid serving " + "stale ownership", + cluster_node_id, + cluster_qvotec_prior_unclean_death() ? ", prior unclean shutdown" : ""), + errhint("Enable cluster.online_join for an online re-declare rejoin (admission " + "self-heal is spec-5.22 follow-up), or cold-restart the cluster after a " + "full clean shutdown. Reads of peer-mastered blocks and non-home work " + "are unaffected."))); + return; + } + + if (cluster_reconfig_bootstrap_quorum_at_initial()) { + /* Clean cold bootstrap: fresh cluster co-booting at INITIAL, ALIVE was + * cleared on the prior clean shutdown (or never written) — no stale + * home blocks. The !prior_unclean_death predicate is already proven by + * the REJOIN arm above (守门裁决 07-15 point 5②). */ + cluster_grd_set_offpath_boot_decided(); + offpath_decided_local = true; + return; + } + + /* UNDECIDED: keep the boot barrier up (fail-closed) and retry next tick. */ +} + + /* ============================================================ * Step 2 D2 — cluster_reconfig_lmon_tick body. * @@ -2208,6 +2330,14 @@ cluster_reconfig_lmon_tick(void) */ cluster_reconfig_joiner_self_tick(); + /* + * Shape A (crash-rejoin re-declare barrier) — the online_join=off + * counterpart: arm the self-fence + close the write gate if THIS node + * crash-rejoined a running cluster, and lift the boot barrier once the + * bootstrap-vs-rejoin classification is proven. No-op on online_join=on. + */ + cluster_reconfig_offpath_rejoin_tick(); + /* * spec-5.15 D1 (INV-J8): the membership-state table — NOT raw CSSD — is the * decision SSOT for the survivor / coordinator set. Maintain it and build diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 777b85586d..478c94c9e1 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -8149,6 +8149,29 @@ cluster_bufmgr_read_block_scn_for_gcs(BufferDesc *buf) return scn; } +/* + * cluster_bufmgr_block_is_extension_for_gcs -- fix 2 (crash-rejoin cold-GRD + * watermark) extension-block whitelist input. + * + * True iff the tag's block number is at or beyond the relation's current + * durable size, i.e. a freshly-extended block that has never been written to + * shared storage (its InvalidScn watermark is correct, so the cold-GRD gate + * must SKIP it, not fail-closed). A pre-existing block (blockNum < nblocks) + * returns false, so an Invalid watermark on it under a self-fence is treated + * as a wiped/cold GRD watermark and fails closed. Invalidates the cached + * size first so a concurrent extension is not missed (cheap: one lseek). + */ +bool +cluster_bufmgr_block_is_extension_for_gcs(BufferTag tag) +{ + SMgrRelation reln = smgropen(BufTagGetRelFileLocator(&tag), InvalidBackendId); + ForkNumber fork = BufTagGetForkNum(&tag); + + /* Drop any cached size so a concurrent extension is not missed. */ + smgrrelease(reln); + return tag.blockNum >= smgrnblocks(reln, fork); +} + bool cluster_bufmgr_refresh_block_from_storage_for_gcs(BufferDesc *buf, SCN *out_page_scn) { diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index 913d670c77..f71ddd9c11 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -1118,6 +1118,51 @@ gcs_block_lost_write_verdict(SCN expected_scn, SCN shipped_scn) return GCS_LOST_WRITE_PASS; } +/* + * fix 2 (crash-rejoin re-declare barrier, defense in depth) — cold-GRD + * watermark verdict. + * + * The storage-fallback / local-master freshness gate normally SKIPs when the + * master pi_watermark_scn is InvalidScn (an old-binary master, a holder + * re-ack whose requester copy is authoritative, or a block that is simply not + * SCN-tracked). But a crash-rejoined node's LOCAL GRD watermark was WIPED by + * the restart, so within an active self-fence an InvalidScn watermark can mask + * a stale home block whose peer holds a newer version — a SKIP there is a + * silent fail-OPEN. This is a second line behind the phase-gate boot barrier, + * which already fences self-home blocks RECOVERING before the acquire reaches + * the freshness gate; it exists so any future path that reaches the freshness + * gate with a wiped watermark still fails closed. + * + * Pure truth table (header-only, unit-testable — no shmem, no I/O): + * expected_scn_valid -> PROVE (run the normal verdict) + * !valid, no self-fence -> SKIP (legit never-tracked / re-ack) + * !valid, self-fence, extension block -> SKIP (genuine new block, never + * cross-node written -> Invalid + * is correct; the storage refresh + * would read past EOF otherwise) + * !valid, self-fence, NOT an extension -> FAIL_CLOSED (wiped/cold GRD watermark on a + * pre-existing block — ambiguous, + * must not serve, Rule 8.A) + */ +typedef enum ClusterColdGrdVerdict { + CLUSTER_COLD_GRD_PROVE, /* watermark valid: run the lost-write verdict */ + CLUSTER_COLD_GRD_SKIP, /* Invalid watermark, provably safe to keep local */ + CLUSTER_COLD_GRD_FAIL_CLOSED, /* Invalid watermark under a self-fence: refuse */ +} ClusterColdGrdVerdict; + +static inline ClusterColdGrdVerdict +cluster_gcs_cold_grd_watermark_verdict(bool expected_scn_valid, bool self_fence_active, + bool is_extension_block) +{ + if (expected_scn_valid) + return CLUSTER_COLD_GRD_PROVE; + if (!self_fence_active) + return CLUSTER_COLD_GRD_SKIP; + if (is_extension_block) + return CLUSTER_COLD_GRD_SKIP; + return CLUSTER_COLD_GRD_FAIL_CLOSED; +} + /* PGRAC: spec-5.2 D2 — read-image intent flag carried in reserved_0[0]. * * When the master forwards an N→S read request to a node that holds the @@ -1913,6 +1958,8 @@ extern ClusterBufmgrGcsDropResult cluster_bufmgr_invalidate_block_for_gcs(Buffer * local copy → PASS keep / stale → refresh + re-verdict → 53R93). */ extern SCN cluster_bufmgr_read_block_scn_for_gcs(BufferDesc *buf); extern bool cluster_bufmgr_refresh_block_from_storage_for_gcs(BufferDesc *buf, SCN *out_page_scn); +/* fix 2 (crash-rejoin cold-GRD watermark) extension-block whitelist input. */ +extern bool cluster_bufmgr_block_is_extension_for_gcs(BufferTag tag); extern void cluster_gcs_block_fallback_verify_refresh(BufferDesc *buf, BufferTag tag, SCN expected_scn); /* PGRAC: spec-5.2 D11 (writer-transfer-revoke) — by-tag local buffer drop diff --git a/src/include/cluster/cluster_grd.h b/src/include/cluster/cluster_grd.h index 0c30376374..8fdd1cc21e 100644 --- a/src/include/cluster/cluster_grd.h +++ b/src/include/cluster/cluster_grd.h @@ -398,6 +398,19 @@ typedef struct ClusterGrdShared { pg_atomic_uint64 join_pcm_fence_member_epoch[CLUSTER_MAX_NODES]; pg_atomic_uint32 recovery_direction; + /* + * TT lane / crash-rejoin re-declare barrier (Shape A) — off-path boot + * barrier. 0 = the online_join=off rejoin/bootstrap decision has NOT + * run this incarnation, so this node cannot prove ownership of its + * home blocks; the phase gate fences self-home blocks RECOVERING until + * it flips (fail-closed boot-race elimination). Set to 1 by the + * off-path rejoin tick once it has DECIDED (crash-rejoin -> also armed + * the self-fence; cold-bootstrap -> no fence). Per-incarnation: + * re-zeroed by the !found shmem init. online_join=on never consults + * it (that path has its own admission + join fence). + */ + pg_atomic_uint32 offpath_boot_decided; + /* spec-5.16 D5 — join-direction remaster counters (dump_grd grd_recovery * segment; kept distinct from the failure-driven remaster_* counters so * ops can tell the two remaster kinds apart — §8 Q6-A). */ @@ -406,6 +419,8 @@ typedef struct ClusterGrdShared { pg_atomic_uint64 join_shards_remastered_count; /* GRD shards moved to joiner */ pg_atomic_uint64 join_block_views_rebuilt_count; /* joiner-home fences lifted */ pg_atomic_uint64 join_block_recovering_failclosed_count; /* 53R9L denied (both gates) */ + pg_atomic_uint64 offpath_crash_rejoin_fenced_count; /* Shape A: off-path crash-rejoin + * fence-arm events (LMON) */ } ClusterGrdShared; /* spec-2.17 D28b — extern atomic generation alloc helper(InitProcess hook). */ @@ -669,6 +684,11 @@ extern void cluster_grd_inc_stale_request_drop(void); extern void cluster_grd_inc_block_path_failclosed(void); /* spec-5.16 D5 — join-direction 53R9L fail-closed bump (requester + master gate). */ extern void cluster_grd_inc_join_block_failclosed(void); +/* Shape A (crash-rejoin re-declare barrier) — off-path boot barrier flag. */ +extern bool cluster_grd_offpath_boot_decided(void); +extern void cluster_grd_set_offpath_boot_decided(void); +extern void cluster_grd_inc_offpath_crash_rejoin_fenced(void); +extern uint64 cluster_grd_offpath_crash_rejoin_fenced_count(void); /* spec-4.6 D5 — bulk snapshot of the 13 grd_recovery counters for the * pg_cluster_state dump (category 'grd_recovery'; one t/249 leg each). */ diff --git a/src/include/cluster/cluster_qvotec.h b/src/include/cluster/cluster_qvotec.h index 0372e63861..a17279bb29 100644 --- a/src/include/cluster/cluster_qvotec.h +++ b/src/include/cluster/cluster_qvotec.h @@ -328,6 +328,9 @@ extern const char *cluster_qvotec_get_collision_state_name(void); * survives Q4 lease expiry can pass the commit gate. * ---------- */ extern bool cluster_qvotec_in_quorum(void); +/* Shape A (crash-rejoin re-declare barrier): prior-incarnation self-slot + * carried ALIVE at startup => this boot follows an UNCLEAN death. */ +extern bool cluster_qvotec_prior_unclean_death(void); /* ---------- diff --git a/src/test/cluster_tap/t/249_grd_recovery_remaster.pl b/src/test/cluster_tap/t/249_grd_recovery_remaster.pl index 1e0847375a..e96c9d5cc1 100644 --- a/src/test/cluster_tap/t/249_grd_recovery_remaster.pl +++ b/src/test/cluster_tap/t/249_grd_recovery_remaster.pl @@ -402,8 +402,8 @@ sub poll_query_until_timeout # ---------- is($pair->node1->safe_psql('postgres', q{SELECT count(*) FROM cluster_dump_state() WHERE category = 'grd_recovery'}), - '31', - 'L7 grd_recovery dump category exposes 31 keys (spec-4.6 D5 + spec-5.16 join + spec-4.6a)'); + '32', + 'L7 grd_recovery dump category exposes 32 keys (spec-4.6 D5 + spec-5.16 join + spec-4.6a + crash-rejoin barrier)'); is($pair->node1->safe_psql('postgres', q{ SELECT string_agg(key, ',' ORDER BY key) FROM cluster_dump_state() @@ -414,7 +414,7 @@ sub poll_query_until_timeout . 'event_old_epoch,holders_rebound,holders_redeclared,' . 'join_block_recovering_failclosed,join_block_views_rebuilt,' . 'join_remaster_done,join_remaster_started,join_shards_remastered,' - . 'last_event_id,rebuild_timeout,remaster_done,remaster_failed,' + . 'last_event_id,offpath_crash_rejoin_fenced,rebuild_timeout,remaster_done,remaster_failed,' . 'remaster_started,shards_remastered,stale_holder_swept,' . 'stale_request_drop,state,state_enum_value,unaffected_holder_survived,' . 'wait_epoch_escape,waiters_requeued', diff --git a/src/test/cluster_tap/t/404_crash_rejoin_stale_read_write_2node.pl b/src/test/cluster_tap/t/404_crash_rejoin_stale_read_write_2node.pl new file mode 100644 index 0000000000..7a9bfb01e3 --- /dev/null +++ b/src/test/cluster_tap/t/404_crash_rejoin_stale_read_write_2node.pl @@ -0,0 +1,336 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# 404_crash_rejoin_stale_read_write_2node.pl +# Crash-rejoin (online_join=off) home-block coherence — the P0 +# fail-open hole the crash-rejoin re-declare barrier (fix 1) and +# the cold-GRD watermark fail-closed (fix 2) close. +# +# Root cause (diagnosed 2026-07-15, evidence bundle bounce-read-p0/): +# with cluster.online_join = off (the DEFAULT), a node that crash- +# restarts into a running cluster skips the joiner self-tick entirely +# (cluster_reconfig.c:1984 `!cluster_online_join -> return`), so its +# self_join_admitted stays at the shmem-init value 1 +# (cluster_reconfig.c:206 `online_join ? 0 : 1`). It boots straight +# to a writable MEMBER with an EMPTY GRD and NO re-declare barrier. +# For blocks whose static hash master is this node +# (cluster_gcs.c:263), the acquire path finds master == self +# (cluster_pcm_lock.c:2565), reads an empty local GRD, and cold-grants +# from the pre-read stale/empty disk page (cluster_pcm_lock.c:2689). +# The round-4c freshness gate that should catch it queries the same +# wiped GRD watermark -> InvalidScn -> SKIP (cluster_gcs_block.c:1278). +# Result: silent stale/empty READ, and a WRITE on the stale image +# that diverges and — after flush ordering — durably LOSES a peer's +# committed rows. Neither ever errors. +# +# Correct behaviour (both fixes): a home block of a just-rejoined node +# is fenced RECOVERING (retryable) until the survivors re-declare what +# they hold; reads/writes are either served COHERENTLY or fail closed +# with a retryable SQLSTATE — NEVER a silent 0-row read and NEVER a +# silently-lost committed write. +# +# L1 ClusterPair startup; a coinciding-filepath home fixture whose +# block0 static master is node1 is selected behaviorally +# L2 node0 writes 64 committed rows (kept in node0 buffers) +# L3 DIAGNOSTIC (never fails): the membership split after node1 bounce +# (node0's view of node1, node1's view of self) — pins the boot +# self-admit path +# L4 HARD (fix contract): node1's post-bounce read of its home table +# is COHERENT (64) or FAILS CLOSED (error) — never a silent 0 +# L5 HARD (fix contract): node1's post-bounce write is COHERENT or +# FAILS CLOSED — never a silent divergent commit +# L6 HARD (blast radius): after node1-first / node0-last flush + a +# full restart, node0's committed rows SURVIVE on disk (no lost +# write); if L5 fenced the write this is trivially satisfied +# L7 fence observability: the join-block fail-closed counter moved +# (fix 1 armed the self-fence) — soft while unimplemented +# +# Spec: spec-5.16 (online rejoin PCM fence) extended to the crash-rejoin / +# online_join=off path; spec-4.7 (survivor re-declare); Rule 8.A. +# +# Author: SqlRush +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../lib"; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::ClusterPair; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'crash_rejoin_coherence', + quorum_voting_disks => 3, + shared_data => 1, + data_port_span => 2, + extra_conf => ['autovacuum = off']); +$pair->start_pair; +usleep(3_000_000); + +ok($pair->wait_for_peer_state(0, 1, 'connected', 60) + && $pair->wait_for_peer_state(1, 0, 'connected', 60), + 'L1 peers connected'); + +sub psql_row +{ + my ($node, $sql) = @_; + my ($rc, $out, $err) = $node->psql('postgres', $sql); + return ($rc, $out, (split(/\n/, $err // ''))[0] // ''); +} + +sub wait_for +{ + my ($cond, $timeout_s, $step_us) = @_; + $step_us //= 500_000; + my $deadline = time() + $timeout_s; + while (time() < $deadline) + { + return 1 if $cond->(); + usleep($step_us); + } + return $cond->() ? 1 : 0; +} + +sub state_val +{ + my ($node, $cat, $key) = @_; + my $v = $node->safe_psql('postgres', + qq{SELECT value FROM pg_cluster_state WHERE category='$cat' AND key='$key'}); + return defined($v) && $v ne '' ? $v + 0 : 0; +} + +sub membership_state +{ + my ($node, $nid) = @_; + my $v = eval { + $node->safe_psql('postgres', + "SELECT state FROM pg_cluster_membership WHERE node_id = $nid"); + }; + return defined($v) ? $v : ''; +} + + +# ============================================================ +# L1b: coinciding-filepath fixture (no shared catalog on this rig, so +# create on both nodes and keep candidates whose relfilenode paths +# coincide -> same shared-storage file). +# ============================================================ +my @cands; +for my $i (1 .. 16) +{ + my $t = "hb_t$i"; + $_->safe_psql('postgres', "CREATE TABLE $t (k int, v int)") + for ($pair->node0, $pair->node1); + my $p0 = $pair->node0->safe_psql('postgres', "SELECT pg_relation_filepath('$t')"); + my $p1 = $pair->node1->safe_psql('postgres', "SELECT pg_relation_filepath('$t')"); + push @cands, $t if ($p0 // '') eq ($p1 // ''); +} +ok(scalar(@cands) > 0, 'L1b coinciding-filepath candidates exist'); + +# ============================================================ +# L2: node0 populates every candidate with 64 committed rows (held in +# node0's buffers until an explicit flush). +# ============================================================ +$pair->node0->safe_psql('postgres', + "INSERT INTO $_ SELECT g, 0 FROM generate_series(1, 64) g") + for @cands; + +# ============================================================ +# CRASH node1 (online_join=off default). stop('immediate') is SIGQUIT +# = an UNCLEAN death: the clean-shutdown ALIVE blank does NOT run, so +# node1's prior voting-disk self-slot keeps CLUSTER_VOTING_SLOT_FLAG_ALIVE +# — the signal the off-path rejoin tick fences on (守门裁决 07-15: ALIVE +# bit, not epoch, because a fast rejoin leaves both sides at epoch INITIAL). +# ============================================================ +$pair->node1->stop('immediate'); +$pair->node1->start; +ok($pair->wait_for_peer_state(0, 1, 'connected', 60) + && $pair->wait_for_peer_state(1, 0, 'connected', 60), + 'L1c node1 re-crashed and transport reconnected'); +usleep(2_000_000); + +# ============================================================ +# L3a HARD: the off-path rejoin tick detected the unclean death and armed +# the fence THIS incarnation (counter is per-incarnation shmem, so a fresh +# value >= 1 proves the ALIVE-bit signal fired). +# ============================================================ +ok(wait_for(sub { + state_val($pair->node1, 'grd_recovery', 'offpath_crash_rejoin_fenced') >= 1 + }, 30), + 'L3a HARD: node1 armed the crash-rejoin fence (offpath_crash_rejoin_fenced >= 1)'); + +# ============================================================ +# L3: DIAGNOSTIC — the membership split (never fails the run; pins the +# self-admit boot path for the report). +# ============================================================ +diag(sprintf('L3 membership after bounce: node0 sees node1=[%s]; ' + . 'node1 sees self=[%s]', + membership_state($pair->node0, 1), membership_state($pair->node1, 1))); +ok(1, 'L3 membership split recorded (diagnostic)'); + +# Behavioral master-discrimination: pick a candidate whose block0 home is +# node1 (post-bounce read is stale/empty or fail-closed) vs node0 (served +# correctly). The vulnerable table drives L4-L6. +my ($vuln, $safe); +for my $t (@cands) +{ + my ($rc, $out, $err) = psql_row($pair->node1, "SELECT count(*) FROM $t"); + diag("probe node1 read $t: rc=$rc out=[$out] err=[$err]"); + if ($rc == 0 && $out eq '0') { $vuln //= $t; } + elsif ($rc == 0 && $out eq '64') { $safe //= $t; } + elsif ($rc != 0) { $vuln //= $t; } # fail-closed also = home block +} +diag("selected vuln=" . ($vuln // 'NONE') . " safe=" . ($safe // 'none')); + +SKIP: { + skip 'no node1-home candidate surfaced this run', 4 unless defined $vuln; + + # ======================================================== + # L4 HARD: node1's read of its home table is COHERENT or FAIL-CLOSED, + # never a silent 0. RED today: rc=0 out=0 (silent empty read). + # ======================================================== + my ($r4, $o4, $e4) = psql_row($pair->node1, "SELECT count(*) FROM $vuln"); + diag("L4 node1 home read: rc=$r4 out=[$o4] err=[$e4]"); + ok(($r4 == 0 && $o4 eq '64') || $r4 != 0, + 'L4 HARD: home-block read is coherent (64) or fail-closed, never a silent 0'); + + # ======================================================== + # L5 HARD: node1's write onto its home block is COHERENT or FAIL- + # CLOSED, never a silently divergent commit. RED today: rc=0, node1 + # then sees 1 row while node0 sees 64 (split brain). + # ======================================================== + my ($wr, $wo, $we) = psql_row($pair->node1, "INSERT INTO $vuln VALUES (1000, 1000)"); + diag("L5 node1 write on home block: rc=$wr err=[$we]"); + # Reads may themselves fail-closed now (node1-home fenced), so use the + # non-dying psql_row; a fenced write ($wr != 0) is coherent by construction. + my (undef, $n1, undef) = psql_row($pair->node1, "SELECT count(*) FROM $vuln"); + my (undef, $n0, undef) = psql_row($pair->node0, "SELECT count(*) FROM $vuln"); + diag("L5 divergence: node1=[$n1] node0=[$n0]"); + # coherent: write refused (fail-closed) OR both nodes agree afterwards. + ok($wr != 0 || ($n1 eq $n0), + 'L5 HARD: home-block write is fail-closed or coherent, never a silent split'); + + # ======================================================== + # L5b HARD (boot-race, Shape A命门): after a CRASH restart, every read + # issued the instant node1 accepts connections must be fail-closed for a + # node1-home table — not one silent 0 may slip through the boot-to- + # decision window. The phase-gate boot barrier (flag defaults 0 at shmem + # init, BEFORE any LMON tick or the qvotec prior_unclean_death probe) + # fences self-home blocks RECOVERING from process start, and the crash- + # rejoin arm then keeps the flag 0, so every read errors 53R9L for the + # whole incarnation. RED on the pre-fix binary (a burst of silent 0s). + # ======================================================== + $pair->node1->stop('immediate'); + $pair->node1->start; # do NOT wait for settle — probe the boot window + my ($silent0, $failclosed, $coherent) = (0, 0, 0); + for my $i (1 .. 40) + { + my ($rc, $out, $err) = psql_row($pair->node1, "SELECT count(*) FROM $vuln"); + if ($rc != 0) { $failclosed++; } + elsif ($out eq '0') { $silent0++; } + else { $coherent++; } # 64 = coherent serve + usleep(150_000); + } + diag("L5b boot-window reads: fail-closed=$failclosed silent0=$silent0 coherent=$coherent"); + is($silent0, 0, + 'L5b HARD: zero silent 0-row reads across the crash-rejoin boot window'); + cmp_ok($failclosed, '>=', 1, 'L5b node1-home reads fail-closed after the crash restart'); + # resettle for L6 + $pair->wait_for_peer_state(0, 1, 'connected', 60); + usleep(1_000_000); + + # ======================================================== + # L6 HARD (blast radius): node1-first / node0-last flush, then a full + # restart — node0's 64 committed rows must SURVIVE on disk. RED + # today: node0=64 wins the flush race but node1's INSERT (if it + # committed on the stale image) is gone AND, worse, the pre-existing + # data can be clobbered; we assert node0's committed rows are intact. + # ======================================================== + $pair->node1->psql('postgres', 'CHECKPOINT'); + $pair->node0->psql('postgres', 'CHECKPOINT'); + $pair->node0->stop('fast'); + $pair->node1->stop('fast'); + $pair->node0->start; + $pair->node1->start; + usleep(3_000_000); + $pair->wait_for_peer_state(0, 1, 'connected', 60); + + my $survive = ''; + for my $i (1 .. 8) + { + my ($rc, $out, $err) = + psql_row($pair->node0, "SELECT count(*) FROM $vuln WHERE v = 0"); + if ($rc == 0) { $survive = $out; last; } + usleep(1_000_000); + } + diag("L6 node0 committed rows after restart: [$survive] (expect >= 64)"); + cmp_ok($survive eq '' ? -1 : $survive + 0, '>=', 64, + "L6 HARD: node0's committed rows survive the crash-rejoin flush race"); +} + +# ============================================================ +# L7 NEGATIVE (full-outage crash co-boot, 守门裁决 07-15 point 4): crash +# BOTH nodes (immediate = ALIVE left set on both), restart both. With +# online_join=off, neither may silently auto-form — each detects its own +# prior unclean death and fences + closes its write gate (53R60). This +# pins the approved semantic consequence: after a full-outage crash the +# cluster waits for spec-5.22 admission / ops, it never self-serves stale. +# (A genuine CLEAN full shutdown clears ALIVE and co-boots normally — that +# is the L6 path above, which served fine.) +# ============================================================ +$pair->node0->stop('immediate'); +$pair->node1->stop('immediate'); +$pair->node0->start; +$pair->node1->start; +usleep(3_000_000); +$pair->wait_for_peer_state(0, 1, 'connected', 60); + +ok(wait_for(sub { + state_val($pair->node0, 'grd_recovery', 'offpath_crash_rejoin_fenced') >= 1 + && state_val($pair->node1, 'grd_recovery', 'offpath_crash_rejoin_fenced') >= 1 + }, 30), + 'L7 NEGATIVE: full-outage crash co-boot fences BOTH nodes (no silent auto-formation)'); + +# Both write gates closed => writes fail-closed 53R60 (retryable), not silent. +{ + my $t = $cands[0]; + my ($w0r, undef, $w0e) = psql_row($pair->node0, "INSERT INTO $t VALUES (2000, 1)"); + my ($w1r, undef, $w1e) = psql_row($pair->node1, "INSERT INTO $t VALUES (2001, 1)"); + diag("L7 post-full-crash writes: node0 rc=$w0r err=[$w0e]; node1 rc=$w1r err=[$w1e]"); + ok($w0r != 0 && $w1r != 0, + 'L7b NEGATIVE: both nodes fail-closed writes after a full-outage crash (53R60/53R9L)'); +} + +# ============================================================ +# L9: a CLEAN full restart clears ALIVE on both nodes, so the co-boot is +# no longer an unclean rejoin — the fence lifts, the cluster serves again, +# and node0's committed rows are intact (the fence is fail-closed, never a +# permanent wedge on a clean restart). +# ============================================================ +$pair->node0->stop('fast'); +$pair->node1->stop('fast'); +$pair->node0->start; +$pair->node1->start; +usleep(3_000_000); +ok($pair->wait_for_peer_state(0, 1, 'connected', 60) + && $pair->wait_for_peer_state(1, 0, 'connected', 60), + 'L9 cluster restarted clean after both legs'); + +my $readback = ''; +for my $i (1 .. 10) +{ + my ($rc, $out, $err) = + psql_row($pair->node0, "SELECT count(*) FROM $cands[0] WHERE v = 0"); + if ($rc == 0) { $readback = $out; last; } + usleep(1_000_000); +} +cmp_ok($readback eq '' ? -1 : $readback + 0, '>=', 64, + 'L9 committed rows intact + cluster usable after a clean restart (fence not permanent)'); + +done_testing(); diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index 1059c08a28..4099ea9c05 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -4020,6 +4020,13 @@ cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out) memset(out, 0, sizeof(*out)); } +/* Shape A stub: dump_grd_recovery emits the crash-rejoin fence counter. */ +uint64 +cluster_grd_offpath_crash_rejoin_fenced_count(void) +{ + return 0; +} + uint32 cluster_grd_outbound_ring_depth(void) { diff --git a/src/test/cluster_unit/test_cluster_gcs_block_lost_write.c b/src/test/cluster_unit/test_cluster_gcs_block_lost_write.c index f8531b4805..831b4a9fdf 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block_lost_write.c +++ b/src/test/cluster_unit/test_cluster_gcs_block_lost_write.c @@ -195,6 +195,41 @@ UT_TEST(test_lost_write_verdict_branches) } +/* + * fix 2 (crash-rejoin cold-GRD watermark) — the pure verdict truth table. + * Positive legs: a valid watermark PROVEs; an Invalid watermark under a + * self-fence FAIL-CLOSEs a pre-existing block. Negative legs: no fence, or a + * genuine extension block, both SKIP (never over-fail-closed). + */ +UT_TEST(test_cold_grd_watermark_verdict_truth_table) +{ + /* valid watermark -> always PROVE regardless of fence / extension */ + UT_ASSERT_EQ((int)cluster_gcs_cold_grd_watermark_verdict(true, false, false), + CLUSTER_COLD_GRD_PROVE); + UT_ASSERT_EQ((int)cluster_gcs_cold_grd_watermark_verdict(true, true, false), + CLUSTER_COLD_GRD_PROVE); + UT_ASSERT_EQ((int)cluster_gcs_cold_grd_watermark_verdict(true, true, true), + CLUSTER_COLD_GRD_PROVE); + + /* NEGATIVE leg 1: Invalid watermark, NO self-fence -> SKIP (legit + * never-tracked / holder re-ack; must not over-fail-closed) */ + UT_ASSERT_EQ((int)cluster_gcs_cold_grd_watermark_verdict(false, false, false), + CLUSTER_COLD_GRD_SKIP); + UT_ASSERT_EQ((int)cluster_gcs_cold_grd_watermark_verdict(false, false, true), + CLUSTER_COLD_GRD_SKIP); + + /* NEGATIVE leg 2: Invalid watermark, self-fence, but a GENUINE extension + * block -> SKIP (never cross-node written; Invalid is correct) */ + UT_ASSERT_EQ((int)cluster_gcs_cold_grd_watermark_verdict(false, true, true), + CLUSTER_COLD_GRD_SKIP); + + /* POSITIVE leg: Invalid watermark, self-fence, pre-existing (non-extension) + * block -> FAIL_CLOSED (wiped/cold GRD watermark, ambiguous, Rule 8.A) */ + UT_ASSERT_EQ((int)cluster_gcs_cold_grd_watermark_verdict(false, true, false), + CLUSTER_COLD_GRD_FAIL_CLOSED); +} + + UT_TEST(test_invalidate_ack_page_scn_helper_round_trip) { GcsBlockInvalidateAckPayload ack; @@ -440,6 +475,7 @@ main(void) UT_RUN(test_forward_payload_expected_pi_watermark_scn_offset_49); UT_RUN(test_expected_pi_watermark_scn_helper_round_trip); UT_RUN(test_lost_write_verdict_branches); + UT_RUN(test_cold_grd_watermark_verdict_truth_table); UT_RUN(test_invalidate_ack_page_scn_helper_round_trip); UT_RUN(test_redeclare_dual_carrier_round_trip); UT_RUN(test_pi_watermark_advance_prototype_linkable); diff --git a/src/test/cluster_unit/test_cluster_reconfig.c b/src/test/cluster_unit/test_cluster_reconfig.c index 49824e87cc..fac8d017e7 100644 --- a/src/test/cluster_unit/test_cluster_reconfig.c +++ b/src/test/cluster_unit/test_cluster_reconfig.c @@ -439,6 +439,32 @@ void cluster_grd_arm_join_pcm_fence(const uint8 *rejoining_set); void cluster_grd_arm_join_pcm_fence(const uint8 *rejoining_set pg_attribute_unused()) {} + +/* Shape A (crash-rejoin re-declare barrier) stubs — the off-path rejoin tick + * references these; the reconfig unit legs do not exercise the crash-rejoin + * arm, so inert stubs suffice. */ +int +cluster_conf_node_count(void) +{ + int n = 0; + int i; + + for (i = 0; i < CLUSTER_MAX_NODES; i++) + if (ut_declared_set[i]) + n++; + return n; +} +bool +cluster_qvotec_prior_unclean_death(void) +{ + return false; +} +void +cluster_grd_set_offpath_boot_decided(void) +{} +void +cluster_grd_inc_offpath_crash_rejoin_fenced(void) +{} /* spec-6.15 D5b: stripe joiner gate (not exercised here) — PROCEED so * the vanilla membership legs run unchanged. */ #include "cluster/cluster_xid_stripe_boot.h"