From df93b68d049b595b7255fb1f5c4ae39c736dc0f4 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 15 Jul 2026 17:11:37 +0800 Subject: [PATCH 1/2] feat(undo): unpin the idle-peer recycle floor and let the cleaner keep pace TT lane (S3 pre-queue baseline forensics, 2026-07-15 leg-1 stop). Two arms, both mechanism, no tuning: H1 idle-unconstrained sentinel. A provably idle node (zero active transactions AND zero held snapshots, both proven in one ProcArrayLock scan) publishes UINT64_MAX as its undo-horizon report instead of its Lamport clock sample. The fold's time-axis min ignores the sentinel structurally, so a lone writer's recycle floor becomes its own local horizon (ms-scale lag) instead of trailing the idle peers' report cadence by 1-2 LMON ticks. At 12k tps that lag exceeded the whole TT slot pool (undo_segments_max_per_instance x 48 = 12,288 slots ~= 1s of allocation), so every pooled slot stayed younger than the floor: zero recyclable inventory, pool camped at the hard cap (11.3k x 53R9E), the "retention horizon pinned" oscillation. The sentinel is capability-gated per connection (new HELLO bit 0x100); an old receiver keeps getting the conservative clock sample, so the mixed-version monotonicity latch can never trip. A sentinel report still owes capability/presence/stability/epoch/freshness proof -- an idle peer that stops reporting stalls the fold exactly as before. A peer waking mid-report-window self-fences through the BELOW_HORIZON read_scn admissibility gate (fail-closed; the wire observe heals the next snapshot): rule 8.A polarity unchanged. Handler-side the sentinel stays OUTSIDE the same-epoch monotone base, so the first real report after waking compares against the last REAL accepted value. H2 pressure-driven continuous recycle. A cleaner pass that exhausts its segment batch while still making progress re-runs immediately instead of sleeping the recycle interval. Pure cadence: the "who may be recycled" decision layer is untouched; a pinned pass (zero progress) or a drained backlog falls back to the interval/stall-retry waits, so the loop cannot busy-spin. Default-GUC honesty (pinned by t/403 L10): the wake-race self-heal above covers snapshot ADMISSIBILITY only. A remote xid whose TT slot was lapped during the storm is NOT self-healing under the default cluster.crossnode_runtime_visibility = off -- node1's read fails closed 53R97 (cluster TT status unknown) with the verdict wire untouched. With the GUC on, the origin-verdict wire (cluster_runtime_visibility.c) resolves the same read through its COMMITTED_BELOW_HORIZON arm. Both behaviours are pinned as a RED->GREEN twin on one driver in t/403 L10. Evidence: t/403 RED on the pre-fix binary with this exact harness -- floor lag 4,122 slots vs a 768-slot pool, 12,352 TT hard-cap failures, the pinned oscillation in the log; GREEN post-fix -- lag_max ~200, zero hard-cap, recycling advances mid-phase, node1's sentinels observable, and a live snapshot on the peer still clamps the floor (protection leg). New TAP t/403 (2-node lone-writer storm + idle peer + wake legs), fold unit rows U17/U17b, undo dump key horizon_idle_sentinel_sent_count (t/214 71->72), HELLO cap ripple (test_cluster_ic), nightly shard stage7-s3-wave 401-403 (also closes the t/401/402 shard hole). Gates: unit clean 178/178, t/017+214+370+402+403 PASS, cluster_regress 13/13, PG 219/219, format/headers/scn-cmp/ges-mode/clog all 0. Spec: spec-5.22e-undo-cluster-retention-horizon.md, spec-3.12, spec-3.13. --- .github/workflows/nightly.yml | 8 + src/backend/cluster/cluster_debug.c | 2 + src/backend/cluster/cluster_ic.c | 4 + src/backend/cluster/cluster_sf_dep.c | 24 + src/backend/cluster/cluster_undo_cleaner.c | 40 +- src/backend/cluster/cluster_undo_horizon_ic.c | 74 ++- src/backend/storage/ipc/procarray.c | 50 ++ src/include/cluster/cluster_ic.h | 14 + src/include/cluster/cluster_sf_dep.h | 4 + src/include/cluster/cluster_undo_horizon.h | 22 + src/include/cluster/cluster_undo_retention.h | 12 + .../t/214_cluster_3_8_undo_lifecycle.pl | 4 +- .../t/403_undo_idle_peer_floor_pin_2node.pl | 524 ++++++++++++++++++ src/test/cluster_unit/test_cluster_debug.c | 5 + src/test/cluster_unit/test_cluster_ic.c | 32 +- .../cluster_unit/test_cluster_undo_horizon.c | 59 +- 16 files changed, 848 insertions(+), 30 deletions(-) create mode 100644 src/test/cluster_tap/t/403_undo_idle_peer_floor_pin_2node.pl diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8cdce481d0..f65e7c4df3 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -239,6 +239,14 @@ jobs: # Own shard: six ClusterPair bring-ups need the wall clock # (L342: every new t/ file lands in a shard the same commit). - { name: stage7-gcs-ownership-gen, ranges: "392-397", unit: false, regress: false } + # t/401 GCS master-direct STALE-ship storage rescue (PR#55) + + # t/402 undo cleaner stall-retry cadence (PR#56) + t/403 TT-lane + # idle-peer floor pin under write load (all 2-node pairs from the + # S3 wave). t/401/402 predate this shard and were in NO CI + # matrix -- closes that hole (L342: every new t/ file lands in a + # shard the same commit). Own shard: three ClusterPair bring-ups + # + the t/403 sustained write phase need the wall clock. + - { name: stage7-s3-wave, ranges: "401-403", unit: false, regress: false } steps: - name: Checkout uses: actions/checkout@v4 diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index fc6d327817..791673930f 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -2877,6 +2877,8 @@ dump_undo(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_undo_horizon_admission_refuse_count())); emit_row(rsinfo, "undo", "horizon_last_floor_scn", fmt_int64((int64)cluster_undo_horizon_last_floor())); + emit_row(rsinfo, "undo", "horizon_idle_sentinel_sent_count", + fmt_int64((int64)cluster_undo_horizon_idle_sentinel_sent_count())); emit_row(rsinfo, "undo", "horizon_peer_reports", cluster_undo_horizon_peer_reports_summary()); emit_row(rsinfo, "undo", "cleaner_header_tt_slots_below_horizon", fmt_int64((int64)cluster_undo_cleaner_header_tt_slots_below_horizon())); diff --git a/src/backend/cluster/cluster_ic.c b/src/backend/cluster/cluster_ic.c index f9cfc75187..1a86edc4f7 100644 --- a/src/backend/cluster/cluster_ic.c +++ b/src/backend/cluster/cluster_ic.c @@ -656,6 +656,10 @@ cluster_ic_build_hello(uint8 out_buf[PGRAC_IC_HELLO_BYTES], uint16 hello_version * negative-ACK protocol capability, same unconditional discipline (see * cluster_ic.h). */ capabilities |= PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1; + /* PGRAC TT lane (S3 idle-peer floor pin): undo-horizon idle-unconstrained + * sentinel protocol capability, same unconditional discipline (see + * cluster_ic.h). */ + capabilities |= PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1; if (capabilities != 0) ic_le_write_uint32(out_buf + PGRAC_IC_HELLO_CAPABILITIES_OFFSET, capabilities); diff --git a/src/backend/cluster/cluster_sf_dep.c b/src/backend/cluster/cluster_sf_dep.c index 9d640eb7a8..9e4d51b6fd 100644 --- a/src/backend/cluster/cluster_sf_dep.c +++ b/src/backend/cluster/cluster_sf_dep.c @@ -235,6 +235,30 @@ cluster_sf_peer_supports_undo_horizon(int32 peer_id) return (capabilities & PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1) != 0; } +/* + * cluster_sf_peer_supports_undo_horizon_idle + * + * TT lane (S3 idle-peer floor pin) capability gate: true iff the peer's + * verified HELLO on the CURRENT connection advertised the idle-unconstrained + * sentinel report bit. Same connection-bound, no-local-GUC discipline as + * the queries above. Consumed by the SENDER only: a provably idle node + * sends the sentinel exclusively to peers that understand it; an old peer + * keeps receiving the conservative clock sample (see cluster_ic.h). + */ +bool +cluster_sf_peer_supports_undo_horizon_idle(int32 peer_id) +{ + uint32 capabilities; + + if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) + return false; + + LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); + capabilities = cluster_sf_peer_cap_bits(&ClusterSfDep->peer_capabilities[peer_id]); + LWLockRelease(&ClusterSfDep->lock); + return (capabilities & PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1) != 0; +} + /* * cluster_sf_peer_supports_gcs_done * diff --git a/src/backend/cluster/cluster_undo_cleaner.c b/src/backend/cluster/cluster_undo_cleaner.c index 1a2fff59f5..e113fa0fe2 100644 --- a/src/backend/cluster/cluster_undo_cleaner.c +++ b/src/backend/cluster/cluster_undo_cleaner.c @@ -364,13 +364,24 @@ undo_cleaner_advance_liveness_tick(void) * cluster-wide COMMITTED recycling for 30-60s). Rule 8.A unchanged: * a retry still recycles ONLY on a proven floor; a persistent cause * keeps every retry stalled. + * + * *out_work_remaining (TT lane, S3 idle-peer floor pin H2): true when + * this pass BOTH consumed its whole segment batch AND made recycle + * progress — the backlog is bigger than one batch, so the caller + * re-runs immediately (pressure-driven continuous mode) instead of + * sleeping the recycle interval. Pure cadence: what may be recycled + * is still decided only by the proven floor above. A pinned pass + * (zero progress) or a drained backlog (batch not exhausted) reports + * false, so the continuous mode can never busy-spin. */ static bool -undo_cleaner_run_pass(void) +undo_cleaner_run_pass(bool *out_work_remaining) { ClusterUndoCleanerPassStats stats; bool floor_retry_needed = false; + *out_work_remaining = false; + if (!cluster_undo_cleaner_enabled) return false; if (!cluster_storage_mode_enabled()) @@ -540,6 +551,17 @@ undo_cleaner_run_pass(void) if (fence_aborted) { floor_retry_needed = true; cluster_undo_horizon_note_pass_abort(); + } else if (batch == 0 + && (stats.segments_marked_recyclable > 0 || stats.shmem_tt_slots_gcd > 0 + || stats.header_tt_slots_below_horizon > 0)) { + /* + * H2 continuous mode: the batch budget ran out while + * recycle progress was still being made -- more eligible + * inventory is waiting than one batch covers. Ask the + * caller to re-run now rather than let a write storm + * outrun the interval cadence. + */ + *out_work_remaining = true; } } } @@ -634,6 +656,7 @@ UndoCleanerMain(void) int rc; int timeout_ms; bool floor_retry; + bool work_remaining; CHECK_FOR_INTERRUPTS(); @@ -649,7 +672,20 @@ UndoCleanerMain(void) CLUSTER_INJECTION_POINT("undo-cleaner-main-loop-iter"); - floor_retry = undo_cleaner_run_pass(); + floor_retry = undo_cleaner_run_pass(&work_remaining); + + /* + * TT lane H2 (pressure-driven continuous mode): a pass that + * exhausted its segment batch while still recycling has a backlog + * bigger than one batch -- under a write storm the interval + * cadence is orders of magnitude too slow (12k xacts/s consume a + * 48-slot segment every ~4ms), so loop straight into the next + * pass. The loop head re-checks interrupts / reload / shutdown, + * and the signal requires PROGRESS, so a pinned or stalled floor + * always falls through to the waits below. + */ + if (work_remaining) + continue; /* * interval 0 = pressure-wakeup only (Q8): block without diff --git a/src/backend/cluster/cluster_undo_horizon_ic.c b/src/backend/cluster/cluster_undo_horizon_ic.c index 1b375d8240..bd30a94d32 100644 --- a/src/backend/cluster/cluster_undo_horizon_ic.c +++ b/src/backend/cluster/cluster_undo_horizon_ic.c @@ -91,8 +91,9 @@ typedef struct ClusterUndoHorizonShmem { pg_atomic_uint64 pass_abort_count; pg_atomic_uint64 wire_reject_count; pg_atomic_uint64 admission_refuse_count; - pg_atomic_uint64 last_floor_scn; /* gauge: last OK fold result */ - pg_atomic_uint64 self_admitted_epoch; /* D5-8: (epoch self last became + pg_atomic_uint64 idle_sentinel_sent_count; /* TT lane: sentinel sends */ + pg_atomic_uint64 last_floor_scn; /* gauge: last OK fold result */ + pg_atomic_uint64 self_admitted_epoch; /* D5-8: (epoch self last became * MEMBER) + 1; 0 = never admitted. * Biased so a cold-formation * admission at epoch 0 is distinct @@ -126,6 +127,7 @@ cluster_undo_horizon_shmem_init(void) pg_atomic_init_u64(&shmem->pass_abort_count, 0); pg_atomic_init_u64(&shmem->wire_reject_count, 0); pg_atomic_init_u64(&shmem->admission_refuse_count, 0); + pg_atomic_init_u64(&shmem->idle_sentinel_sent_count, 0); pg_atomic_init_u64(&shmem->last_floor_scn, 0); pg_atomic_init_u64(&shmem->self_admitted_epoch, 0); } @@ -239,18 +241,33 @@ cluster_undo_horizon_report_handler(const ClusterICEnvelope *env, const void *pa * snapshot BELOW the accepted bound exists over there -- latch the * violation (the fold stalls, U14) and do not move the payload. A * later conforming report clears the latch. + * + * The idle-unconstrained sentinel (TT lane) is OUTSIDE the monotone + * sequence: it claims "no constraint", not a bound, so it neither + * regresses against the accepted base (its scn_local is the ceiling + * anyway) nor BECOMES the base -- otherwise the sender's first real + * report after waking (its clock, numerically below the sentinel) + * would latch a spurious regression and stall the fold on every + * idle->active transition. A real report after a sentinel still + * compares against the last REAL accepted value (the sender's clock + * is monotone, so that comparison holds across the idle gap). */ - if (slot->accepted_any && slot->accepted_epoch == wire.epoch - && scn_time_cmp((SCN)wire.horizon_scn, (SCN)slot->accepted_scn) < 0) { - undo_horizon_reject("same-epoch regression", sender); - undo_horizon_slot_publish(slot, &wire, now_us, true); + if ((SCN)wire.horizon_scn != CLUSTER_UNDO_HORIZON_REPORT_UNCONSTRAINED) { + if (slot->accepted_any && slot->accepted_epoch == wire.epoch + && scn_time_cmp((SCN)wire.horizon_scn, (SCN)slot->accepted_scn) < 0) { + undo_horizon_reject("same-epoch regression", sender); + undo_horizon_slot_publish(slot, &wire, now_us, true); + return; + } + + undo_horizon_slot_publish(slot, &wire, now_us, false); + slot->accepted_scn = wire.horizon_scn; + slot->accepted_epoch = wire.epoch; + slot->accepted_any = true; return; } undo_horizon_slot_publish(slot, &wire, now_us, false); - slot->accepted_scn = wire.horizon_scn; - slot->accepted_epoch = wire.epoch; - slot->accepted_any = true; } void @@ -305,6 +322,7 @@ cluster_undo_horizon_lmon_tick(void) ClusterUndoHorizonWire wire; uint64 now_us; SCN report; + bool unconstrained; int pi; if (!cluster_enabled || cluster_node_id < 0 || UndoHorizonShmem == NULL) @@ -320,11 +338,25 @@ cluster_undo_horizon_lmon_tick(void) return; last_sent_us = now_us; + /* + * TT lane (S3 idle-peer floor pin): a PROVABLY idle node (zero active + * xacts AND zero held snapshots) reports the unconstrained sentinel so + * its clock-sample lag stops pinning a lone writer's recycle floor. + * MRP-active standbys are never "idle" in this sense (their future + * snapshots read at consistent_scn, S3.0 row 4), and any uncertainty + * in the probe answers false -- the conservative sample then flows + * exactly as before. The probe runs AFTER the sample: a snapshot + * appearing in between flips it to false (conservative); one + * disappearing in between just means the constraint is gone. + */ + unconstrained = !cluster_mrp_should_start() && cluster_undo_retention_all_quiescent(); + wire.epoch = cluster_epoch_get_current(); - wire.horizon_scn = (uint64)report; wire.sender_interval_ms = (uint32)cluster_lmon_main_loop_interval; for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) { + bool sentinel; + if (pi == cluster_node_id) continue; if (cluster_conf_lookup_node(pi) == NULL) @@ -340,6 +372,18 @@ cluster_undo_horizon_lmon_tick(void) if (!cluster_sf_peer_supports_undo_horizon(pi)) continue; + /* + * Sentinel send-side hard gate: an old receiver would fold the + * raw sentinel value harmlessly but latch a spurious same-epoch + * regression when this node wakes, so it keeps getting the + * conservative clock sample instead (see cluster_ic.h). + */ + sentinel = unconstrained && cluster_sf_peer_supports_undo_horizon_idle(pi); + wire.horizon_scn + = sentinel ? (uint64)CLUSTER_UNDO_HORIZON_REPORT_UNCONSTRAINED : (uint64)report; + if (sentinel) + pg_atomic_fetch_add_u64(&UndoHorizonShmem->idle_sentinel_sent_count, 1); + (void)cluster_ic_send_envelope(PGRAC_IC_MSG_UNDO_HORIZON, pi, &wire, sizeof(wire)); /* fire-and-forget (L456): transport retention / errors surface * through the transport's own paths; a lost report just ages the @@ -634,6 +678,16 @@ UNDO_HORIZON_NOTE(pass_abort) UNDO_HORIZON_NOTE(wire_reject) UNDO_HORIZON_NOTE(admission_refuse) +/* TT lane: sender-side sentinel gauge (bumped inline in the tick; only the + * reader half of the NOTE pattern is needed). */ +uint64 +cluster_undo_horizon_idle_sentinel_sent_count(void) +{ + return UndoHorizonShmem == NULL + ? 0 + : pg_atomic_read_u64(&UndoHorizonShmem->idle_sentinel_sent_count); +} + void cluster_undo_horizon_note_floor(SCN scn) { diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c index a741bab4de..d13d87c524 100644 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@ -2264,6 +2264,56 @@ cluster_undo_retention_horizon(void) return SCN_VALID(min) ? min : cluster_scn_current(); } + +/* + * cluster_undo_retention_all_quiescent -- TT lane (S3 idle-peer floor pin). + * + * True iff this node provably holds no undo retention constraint at this + * instant: no ProcArray entry has an assigned xid, an xmin, or a published + * CLUSTER read_scn. Both predicate legs (zero active transactions AND + * zero held snapshots) are required; any doubt answers false and the + * horizon-report sender falls back to the conservative clock sample. + * + * Prepared transactions' dummy PGPROCs sit in the ProcArray with a valid + * xid, so an unresolved prepared xact makes this node non-quiescent — the + * conservative direction (its undo must stay reachable). Walsenders / + * background workers with an xmin likewise read as constraints. + * + * Same locking contract as cluster_undo_retention_horizon() above: SHARED + * ProcArrayLock, never called under seg->lock / undo lifecycle_lock (C17). + * The xid reads follow the ComputeXidHorizons dense-array access pattern. + */ +bool +cluster_undo_retention_all_quiescent(void) +{ + ProcArrayStruct *arrayP = procArray; + TransactionId *other_xids = ProcGlobal->xids; + bool quiescent = true; + int index; + + if (cluster_node_id < 0) + return false; /* cluster disabled: never claim idle */ + + LWLockAcquire(ProcArrayLock, LW_SHARED); + for (index = 0; index < arrayP->numProcs; index++) + { + int pgprocno = arrayP->pgprocnos[index]; + PGPROC *proc = &allProcs[pgprocno]; + TransactionId xid = UINT32_ACCESS_ONCE(other_xids[index]); + TransactionId xmin = UINT32_ACCESS_ONCE(proc->xmin); + SCN r = (SCN) pg_atomic_read_u64(&proc->cluster_read_scn_atomic); + + if (TransactionIdIsValid(xid) || TransactionIdIsValid(xmin) + || SCN_VALID(r)) + { + quiescent = false; + break; + } + } + LWLockRelease(ProcArrayLock); + + return quiescent; +} #endif /* diff --git a/src/include/cluster/cluster_ic.h b/src/include/cluster/cluster_ic.h index e240885523..fa74019724 100644 --- a/src/include/cluster/cluster_ic.h +++ b/src/include/cluster/cluster_ic.h @@ -350,6 +350,20 @@ typedef enum ClusterICPlane { * the pre-BUSY protocol. Timeout stays the backstop for packet loss / dead * nodes. */ #define PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 ((uint32)0x00000080U) +/* PGRAC: TT lane (S3 idle-peer floor pin) — this binary understands the + * UNDO_HORIZON idle-unconstrained sentinel report (horizon_scn == + * UINT64_MAX): a provably idle sender (zero active xacts AND zero held + * snapshots) publishes "no retention constraint" instead of its clock + * sample, so a lone writer's recycle floor no longer trails the idle + * peers' report cadence (the S3 floor-lag pin: lag x write-rate exceeded + * the whole TT slot pool). A PROTOCOL capability, advertised + * unconditionally. Send-side hard gate: the sentinel is only sent to a + * peer whose CURRENT connection advertised this bit — an old receiver + * would fold the raw sentinel VALUE harmlessly (a max value never wins a + * min fold) but would latch a spurious same-epoch REGRESSION stall when + * the sender wakes and reports a real value again, so old peers keep + * receiving the conservative clock sample instead. */ +#define PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 ((uint32)0x00000100U) /* * PGRAC: spec-7.2 D2 — plane + connection-epoch ride the documented-zero * pad region (capabilities precedent: occupy pad bytes, do not resize V1). diff --git a/src/include/cluster/cluster_sf_dep.h b/src/include/cluster/cluster_sf_dep.h index 2cd33bc83f..a83cae98af 100644 --- a/src/include/cluster/cluster_sf_dep.h +++ b/src/include/cluster/cluster_sf_dep.h @@ -201,6 +201,10 @@ extern bool cluster_peer_supports_undo_authority_serve(int32 peer_id); /* spec-5.22e D5-2: undo-horizon report capability (connection-bound; see * cluster_sf_note_peer_disconnected_gen) + the close-funnel reset hooks. */ extern bool cluster_sf_peer_supports_undo_horizon(int32 peer_id); +/* TT lane (S3 idle-peer floor pin): idle-unconstrained sentinel report + * capability (connection-bound, same discipline). SENDER-side gate only: + * an old peer keeps receiving the conservative clock sample. */ +extern bool cluster_sf_peer_supports_undo_horizon_idle(int32 peer_id); /* GCS-race round-2 review F6: GCS completion-proof capability * (connection-bound, same discipline). An unknown/old/reconnecting peer * reads false, which every consumer treats in the SAFE direction: the diff --git a/src/include/cluster/cluster_undo_horizon.h b/src/include/cluster/cluster_undo_horizon.h index d346efb9ee..052da9bab7 100644 --- a/src/include/cluster/cluster_undo_horizon.h +++ b/src/include/cluster/cluster_undo_horizon.h @@ -55,6 +55,25 @@ #define CLUSTER_UNDO_HORIZON_INTERVAL_MIN_MS 100 #define CLUSTER_UNDO_HORIZON_INTERVAL_MAX_MS 60000 +/* + * Idle-unconstrained sentinel report value (TT lane, S3 idle-peer floor + * pin). A PROVABLY idle sender (cluster_undo_retention_all_quiescent(): + * zero active xacts AND zero held snapshots) publishes this instead of its + * clock sample, telling the fold "I hold no retention constraint": the + * time-axis-min fold ignores it structurally (scn_local == SCN_MAX_LOCAL + * never wins a min), so a lone writer's floor becomes its own local + * horizon instead of trailing the idle peers' report cadence. All other + * proof obligations (capability, presence, stability, epoch, freshness) + * still apply to a sentinel report — an idle peer that stops reporting + * still stalls the fold. A sender that wakes mid-report-window self-fences + * through the BELOW_HORIZON read_scn admissibility gate (fail-closed, + * observe heals the next snapshot), so rule 8.A polarity is unchanged. + * The value deliberately has all node bits set (node id 255 is invalid) + * and local == SCN_MAX_LOCAL (the wraparound-guard ceiling): no lawful + * clock sample can ever collide with it. + */ +#define CLUSTER_UNDO_HORIZON_REPORT_UNCONSTRAINED ((SCN)UINT64_MAX) + typedef enum ClusterUndoHorizonFoldStatus { CLUSTER_UNDO_HORIZON_FOLD_OK = 0, CLUSTER_UNDO_HORIZON_FOLD_STALLED @@ -208,6 +227,9 @@ extern void cluster_undo_horizon_note_wire_reject(void); extern uint64 cluster_undo_horizon_wire_reject_count(void); extern void cluster_undo_horizon_note_admission_refuse(void); extern uint64 cluster_undo_horizon_admission_refuse_count(void); +/* TT lane: idle-unconstrained sentinel reports sent (sender-side gauge of + * the provably-idle predicate actually firing; one bump per peer send). */ +extern uint64 cluster_undo_horizon_idle_sentinel_sent_count(void); extern const char *cluster_undo_horizon_peer_reports_summary(void); extern void cluster_undo_horizon_note_floor(SCN scn); extern SCN cluster_undo_horizon_last_floor(void); diff --git a/src/include/cluster/cluster_undo_retention.h b/src/include/cluster/cluster_undo_retention.h index ed54f6d7e9..f0e4dbc78b 100644 --- a/src/include/cluster/cluster_undo_retention.h +++ b/src/include/cluster/cluster_undo_retention.h @@ -48,6 +48,18 @@ */ extern SCN cluster_undo_retention_horizon(void); +/* + * cluster_undo_retention_all_quiescent -- TT lane (S3 idle-peer floor pin): + * true iff this node PROVABLY holds no retention constraint right now: + * every ProcArray entry has no assigned xid, no xmin and no published + * CLUSTER read_scn (zero active transactions AND zero held snapshots — + * both legs required). Anything uncertain (cluster disabled, any live + * xid/xmin/read_scn, prepared-xact dummy PGPROCs with an xid) returns + * false so the caller falls back to the conservative clock sample. Same + * locking contract as cluster_undo_retention_horizon(). + */ +extern bool cluster_undo_retention_all_quiescent(void); + /* * Pure retention predicates (spec-3.12 D2/D3). No shmem / no LWLock / no I/O; diff --git a/src/test/cluster_tap/t/214_cluster_3_8_undo_lifecycle.pl b/src/test/cluster_tap/t/214_cluster_3_8_undo_lifecycle.pl index 51a1674f88..39ff5ae29f 100644 --- a/src/test/cluster_tap/t/214_cluster_3_8_undo_lifecycle.pl +++ b/src/test/cluster_tap/t/214_cluster_3_8_undo_lifecycle.pl @@ -87,8 +87,8 @@ # ---------- my $undo_row_count = $node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='undo'}); -is($undo_row_count, '71', - "L2 undo category has 71 rows (5 record + 4 lifecycle + 3 fsync + 4 smgr + 5 durable-tt + 5 retention + 9 terminal-authority [spec-6.2] + 6 cleaner + 4 buf/extent obs [spec-3.18 D7] + 1 spec-3.22 retention_off_recycle + 4 checkpoint-writeback boundary [spec-4.8ab D7] + 2 record-segment reclaim [spec-4.12a D5] + 1 residual-revalidate-drop [spec-4.12a Hardening v1.0.1] + 1 retention_max_recycle_horizon [spec-6.15 D4] + 6 undo GCS grant-plane [spec-5.22b D2-6] + 8 horizon brake observability [spec-5.22e D5-5, D5 lane 漏更 t/214] + 3 TT-rollover failure split [S3 forensics step 1a/1b])" +is($undo_row_count, '72', + "L2 undo category has 72 rows (5 record + 4 lifecycle + 3 fsync + 4 smgr + 5 durable-tt + 5 retention + 9 terminal-authority [spec-6.2] + 6 cleaner + 4 buf/extent obs [spec-3.18 D7] + 1 spec-3.22 retention_off_recycle + 4 checkpoint-writeback boundary [spec-4.8ab D7] + 2 record-segment reclaim [spec-4.12a D5] + 1 residual-revalidate-drop [spec-4.12a Hardening v1.0.1] + 1 retention_max_recycle_horizon [spec-6.15 D4] + 6 undo GCS grant-plane [spec-5.22b D2-6] + 8 horizon brake observability [spec-5.22e D5-5, D5 lane 漏更 t/214] + 3 TT-rollover failure split [S3 forensics step 1a/1b] + 1 idle-sentinel gauge [TT lane, S3 floor pin])" ); diff --git a/src/test/cluster_tap/t/403_undo_idle_peer_floor_pin_2node.pl b/src/test/cluster_tap/t/403_undo_idle_peer_floor_pin_2node.pl new file mode 100644 index 0000000000..fabe9db532 --- /dev/null +++ b/src/test/cluster_tap/t/403_undo_idle_peer_floor_pin_2node.pl @@ -0,0 +1,524 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# 403_undo_idle_peer_floor_pin_2node.pl +# TT lane — idle-peer floor pin + recycle cadence under write load. +# +# Root cause under repair (S3 pre-queue baseline 2026-07-15, node0 +# 1.46M lock-timeout + 11.3k 53R9E at segments 255/256 + repeated +# "retention horizon pinned" with the floor PROVEN): the cluster +# recycle floor folds in every idle peer's horizon report, and an +# idle peer reports its own Lamport clock sample. That sample only +# tracks the writer through envelope observes at the heartbeat / +# report cadence (~1-2 LMON ticks), so the proven floor trails the +# writer's clock by 1-2 ticks. The TT slot pool is +# undo_segments_max_per_instance x 48 slots; as soon as +# write-rate x floor-lag exceeds the pool, EVERY pooled slot is +# younger than the floor, nothing is ever recyclable, and the pool +# camps at the hard cap while the pinned/progress cycle oscillates. +# +# The fix (two pre-approved arms): +# H1 a provably idle peer (zero active xacts AND zero held +# snapshots) reports an UNCONSTRAINED sentinel instead of its +# clock sample; the fold's min skips it, so a lone writer's +# floor is its own local horizon (ms-scale lag). Anything +# not provably idle keeps today's conservative sample; a peer +# waking mid-window self-fences through the BELOW_HORIZON +# read_scn admissibility gate (fail-closed, retry heals). +# H2 a pass that exhausts its segment batch while making +# progress re-runs immediately instead of sleeping the +# recycle interval (pressure-driven continuous mode); the +# "who may be recycled" decision layer is untouched. +# +# The lmon interval is raised to 2s as a deterministic lag +# amplifier and the pool is shrunk to the GUC floor (16 segments = +# 768 slots) so the pin reproduces within seconds at single-psql +# write rates; budgets keep the whole file under ~90s of load. +# +# L1 ClusterPair startup; initial cluster floor proven +# L2 write phase: node0 sustained short transactions, node1 +# strictly idle; per-sample floor lag + recycle progress +# L3 HARD ASSERT (H1 contract): max observed (writer clock - +# proven floor) stays below the pool capacity in slots +# L4 HARD ASSERT: zero hard-cap failures across the whole phase +# (undo.tt_rollover_fail_hard_cap_count et al stay 0) +# L5 HARD ASSERT (H2 contract): recycling makes progress DURING +# the phase (segments_marked_recyclable advances), not only +# after quiesce +# L6 sentinel evidence: node1 published idle sentinels and node0's +# fold never stalled during the phase +# L7 PROTECTION NEGATIVE: a live snapshot on node1 suppresses the +# sentinel — the floor clamps to the holder while it lives and +# resumes after release (predicate never fires when unsure) +# L8 wake-up self-heal: node1's first cross-node read after the +# storm succeeds (bounded retries allowed: the inadmissible +# BELOW_HORIZON arm is fail-closed and observe heals it) +# L9 restart clean +# +# Spec: spec-5.22e (cluster undo retention brake) S2.1 sampling rule + +# S3 pre-queue baseline forensics (TT lane, idle-peer floor pin). +# +# Author: SqlRush +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../lib"; + +use File::Temp qw(tempdir); +use IPC::Run (); +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::ClusterPair; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep time); + +my $POOL_SEGMENTS = 16; # GUC floor; smallest legal pool +my $SLOTS_PER_SEG = 48; # TT_SLOTS_PER_SEGMENT (ABI const) +my $POOL_SLOTS = $POOL_SEGMENTS * $SLOTS_PER_SEG; +my $LMON_INTERVAL_S = 2; # lag amplifier (freshness = 3x) +my $PHASE_S = 60; # FIXED write-phase duration; the + # writers are fed more work than + # they can finish and are killed + # at the deadline, so machine + # speed variance cannot flake a + # completion budget +my $WRITER_LINES = 120_000; # per writer; >> any 60s throughput +my $SCN_LOCAL_MASK = '72057594037927936'; # 2^56 (numeric, server-side) + +my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'undo_floor_pin', + quorum_voting_disks => 3, + shared_data => 1, + data_port_span => 2, + extra_conf => [ + 'autovacuum = off', + 'synchronous_commit = off', + 'cluster.cssd_heartbeat_interval_ms = 2000', + 'cluster.cssd_dead_deadband_factor = 10', + "cluster.lmon_main_loop_interval = ${\($LMON_INTERVAL_S * 1000)}", + "cluster.undo_segments_max_per_instance = $POOL_SEGMENTS", + 'cluster.undo_cleaner_interval_ms = 1000', + ]); +$pair->start_pair; + +usleep(3_000_000); + +is($pair->node0->safe_psql('postgres', 'SELECT 1'), + '1', 'L1 node0 postmaster alive'); +is($pair->node1->safe_psql('postgres', 'SELECT 1'), + '1', 'L1 node1 postmaster alive'); + +ok($pair->wait_for_peer_state(0, 1, 'connected', 60) + && $pair->wait_for_peer_state(1, 0, 'connected', 60), + 'L1 peers connected'); + + +sub state_val +{ + my ($node, $cat, $key) = @_; + my $v = $node->safe_psql('postgres', + qq{SELECT value FROM pg_cluster_state WHERE category='$cat' AND key='$key'}); + return defined($v) && $v ne '' ? $v + 0 : 0; +} + +sub wait_for +{ + my ($cond, $timeout_s, $step_us) = @_; + $step_us //= 500_000; + my $deadline = time() + $timeout_s; + while (time() < $deadline) + { + return 1 if $cond->(); + usleep($step_us); + } + return $cond->() ? 1 : 0; +} + +# Retry-tolerant single-statement write. Cross-node block ping-pong can +# transiently bounce a statement off the GES master work queue +# (source=master-reject-queue-full, elapsed_ms=0 -- a capacity reject +# mapped to the retryable 53R70 shape, structural fix owned by the +# convert-queue lane); the harness only needs SOME commits to land, so it +# retries instead of dying mid-leg. +sub write_retry +{ + my ($node, $sql, $times) = @_; + for my $i (1 .. $times) + { + for my $try (1 .. 3) + { + my ($rc, $out, $err) = $node->psql('postgres', $sql); + last if $rc == 0; + usleep(200_000); + } + } +} + +# Time-axis (local) part of the proven floor, server-side (uint64-safe). +# The encoded floor carries the winning reporter's NODE bits in the high +# byte, so ORDER comparisons on the raw value are meaningless when the +# winner changes (the L457 raw-SCN-comparison ban applies to the harness +# too); every before/after comparison below uses this projection. +sub floor_local +{ + my ($node) = @_; + my $v = $node->safe_psql('postgres', qq{ + SELECT (value::numeric % ${SCN_LOCAL_MASK}::numeric)::text + FROM pg_cluster_state + WHERE category='undo' AND key='horizon_last_floor_scn'}); + return defined($v) && $v ne '' ? $v + 0 : 0; +} + +# Floor lag in local-SCN units, computed SERVER-side (uint64-safe: Perl NV +# would lose precision on the node-tagged encoded floor). One statement = +# one snapshot, so the pair is self-consistent. Returns -1 while the floor +# is unproven (never counted as a lag sample). +sub floor_lag +{ + my ($node) = @_; + my $v = $node->safe_psql('postgres', qq{ + SELECT CASE WHEN f.v = 0 THEN -1 + ELSE (s.v - (f.v % ${SCN_LOCAL_MASK}::numeric))::text::numeric END + FROM (SELECT value::numeric AS v FROM pg_cluster_state + WHERE category='scn' AND key='scn_current_local') s, + (SELECT value::numeric AS v FROM pg_cluster_state + WHERE category='undo' AND key='horizon_last_floor_scn') f}); + return defined($v) && $v ne '' ? $v + 0 : -1; +} + +# ============================================================ +# L1b: fixture + initial floor proven before the load phase. +# +# Coinciding-filepath heap fixture on shared storage (t/394 recipe): +# this rig has no shared catalog, so the table is created on BOTH +# nodes and candidates are retried until the relfilenode paths +# coincide -- then both catalogs address the same shared-storage +# file and node1's L8 read is a true cross-node MVCC read of +# node0's writes. +# +# The fixture commits also prime the SCN clock: a fully idle fresh +# formation has never advanced it, so the local horizon sample is +# InvalidScn and the fold self-stalls MALFORMED (U7b) until the +# first commit. +# ============================================================ +my $tbl; +for my $i (1 .. 12) +{ + my $t = "t403_$i"; + $_->safe_psql('postgres', "CREATE TABLE $t (k int, v int)") + for ($pair->node0, $pair->node1); + my $p0 = $pair->node0->safe_psql('postgres', "SELECT pg_relation_filepath('$t')"); + my $p1 = $pair->node1->safe_psql('postgres', "SELECT pg_relation_filepath('$t')"); + if (($p0 // '') eq ($p1 // '')) { $tbl = $t; last; } +} +die 'no coinciding filepath found' unless defined $tbl; +diag("fixture table=$tbl"); +$pair->node0->safe_psql('postgres', + "INSERT INTO $tbl SELECT g, 0 FROM generate_series(1, 64) g"); + +ok(wait_for(sub { state_val($pair->node0, 'undo', 'horizon_last_floor_scn') > 0 }, 30), + 'L1b initial cluster floor proven on node0'); + + +# ============================================================ +# L2: write phase. node0 runs a sustained stream of short +# autocommit transactions from 2 background psql sessions; node1 is +# NEVER touched during the phase (its idleness is the mechanism +# under test). All sampling runs against node0 only. +# ============================================================ + +my $tmpdir = tempdir(CLEANUP => 1); +my $sqlfile = "$tmpdir/writer.sql"; +{ + open(my $fh, '>', $sqlfile) or die "cannot write $sqlfile: $!"; + for my $i (1 .. $WRITER_LINES) + { + my $k = ($i % 64) + 1; + print $fh "UPDATE $tbl SET v = v + 1 WHERE k = $k;\n"; + } + close($fh); +} + +my $stall_pre = state_val($pair->node0, 'undo', 'horizon_stall_count'); +my $recyc_pre = state_val($pair->node0, 'undo', 'cleaner_segments_marked_recyclable'); +my $hardcap_tt_pre = state_val($pair->node0, 'undo', 'tt_rollover_fail_hard_cap_count'); +my $hardcap_seg_pre = state_val($pair->node0, 'undo', 'segment_hard_cap_fail_count'); +my $rollover_pre = state_val($pair->node0, 'undo', 'tt_retention_rollover_count'); + +my @writers; +for my $w (1 .. 2) +{ + my ($in, $out, $err) = ('', '', ''); + push @writers, + [ + IPC::Run::start( + [ + 'psql', '-X', '-A', '-t', '-q', + '-d', $pair->node0->connstr('postgres'), + '-f', $sqlfile + ], + \$in, \$out, \$err), + \$out, \$err + ]; +} + +# Sampling loop: floor lag + mid-phase recycle progress, ~2 samples/s, +# for a FIXED phase duration; the writers are then killed (they carry far +# more work than any machine finishes in the window, so wall-clock speed +# never flakes this leg). +my $lag_max = 0; +my $recyc_mid_delta = 0; +my $phase_t0 = time(); +while (time() - $phase_t0 < $PHASE_S) +{ + my $lag = floor_lag($pair->node0); + $lag_max = $lag if $lag > $lag_max; + my $rd = state_val($pair->node0, 'undo', 'cleaner_segments_marked_recyclable') + - $recyc_pre; + $recyc_mid_delta = $rd if $rd > $recyc_mid_delta; + + usleep(400_000); +} +$_->[0]->kill_kill for @writers; + +my $writer_errs = ''; +$writer_errs .= ${ $_->[2] } for @writers; +my $rollover_delta = state_val($pair->node0, 'undo', 'tt_retention_rollover_count') + - $rollover_pre; +diag(sprintf( + "L2 phase: %ds fixed, lag_max=%d slots-units (pool=%d), " + . "mid-phase recyclable delta=%d, rollovers=%d", + $PHASE_S, $lag_max, $POOL_SLOTS, $recyc_mid_delta, $rollover_delta)); + +# >= 2 pool laps of slot churn proves the writers ran sustained, unwedged. +cmp_ok($rollover_delta, '>=', 2 * $POOL_SEGMENTS, + 'L2 write phase sustained (>= 2 pool laps of TT rollover)'); +cmp_ok($lag_max, '>', 0, 'L2b lag sampler captured real samples'); + +# Post-storm GES drain barrier. The storm leaves node1's master work +# queue saturated (source=master-reject-queue-full, elapsed_ms=0) for a +# bounded tail while admitted waiters expire their 60s timeouts. The +# legs below measure floor semantics, not queue capacity (the +# convert-queue lane owns that wedge class), so wait for the lock plane +# to serve again -- with a HARD budget: a queue that never drains is a +# wedge this test must fail on, not paper over. +my $drained = 0; +my $drain_t0 = time(); +for my $i (1 .. 45) +{ + my ($rc, $out, $err) = + $pair->node0->psql('postgres', "UPDATE $tbl SET v = v + 1 WHERE k = 64"); + if ($rc == 0) { $drained = 1; last; } + usleep(2_000_000); +} +ok($drained, + sprintf('L2c post-storm GES plane drained (%.0fs)', time() - $drain_t0)); + +# ============================================================ +# L3: HARD ASSERT (H1 contract) — the proven floor tracks the lone +# writer's clock: the worst observed lag stays below the pool +# capacity, so retained-younger-than-floor inventory can never fill +# the pool. RED on the pre-fix binary: the floor trails the idle +# peers' report cadence (1-2 lmon ticks x write rate >> pool). +# ============================================================ +cmp_ok($lag_max, '<', $POOL_SLOTS, + 'L3 HARD ASSERT: floor lag stays below pool capacity under a lone writer'); + +# ============================================================ +# L4: HARD ASSERT — the pool never hit the hard cap. +# ============================================================ +my $hardcap_tt = state_val($pair->node0, 'undo', 'tt_rollover_fail_hard_cap_count') + - $hardcap_tt_pre; +my $hardcap_seg = state_val($pair->node0, 'undo', 'segment_hard_cap_fail_count') + - $hardcap_seg_pre; +is($hardcap_tt, 0, + 'L4 HARD ASSERT: zero TT rollover hard-cap failures during the phase'); +is($hardcap_seg, 0, 'L4b zero record-extent hard-cap failures during the phase'); +unlike($writer_errs, qr/hard cap|53R9E/i, + 'L4c writers saw no hard-cap errors'); + +# ============================================================ +# L5: HARD ASSERT (H2 contract) — recycling progressed DURING the +# phase (not only at quiesce): the cleaner kept pace with the +# writer instead of camping on the interval. +# ============================================================ +cmp_ok($recyc_mid_delta, '>', 0, + 'L5 HARD ASSERT: segments marked recyclable while the write phase ran'); + +# ============================================================ +# L6: sentinel evidence. node1 (idle throughout) published +# unconstrained sentinel reports; node0's fold consumed them without +# a single stall. (node1 is only queried AFTER the phase, so the +# query itself cannot have broken its idleness mid-phase.) +# ============================================================ +cmp_ok(state_val($pair->node1, 'undo', 'horizon_idle_sentinel_sent_count'), + '>', 0, 'L6 node1 sent idle-unconstrained sentinel reports'); +is(state_val($pair->node0, 'undo', 'horizon_stall_count') - $stall_pre, + 0, 'L6b node0 fold never stalled during the phase'); + +# ============================================================ +# L7: PROTECTION NEGATIVE — a live snapshot on node1 suppresses the +# sentinel. While node1 holds a REPEATABLE READ snapshot, its +# report is the conservative sample again, so node0's floor clamps +# at the holder and stops tracking new commits; releasing the +# snapshot lets the floor move again. This pins the predicate's +# fail-conservative direction (rule 8.A polarity). +# ============================================================ +my $sentinel_pre_hold = + state_val($pair->node1, 'undo', 'horizon_idle_sentinel_sent_count'); + +# The snapshot is taken with SELECT 1 -- deliberately NO table access, so +# the holder cannot bounce off a transient cross-node GES reject and die +# early (which would silently un-hold the snapshot and flake both asserts +# below). A cluster-storage-mode snapshot publishes read_scn regardless +# of what it reads. +my ($h_in, $h_out, $h_err) = ('', '', ''); +my $holder = IPC::Run::start( + [ + 'psql', '-X', '-A', '-t', '-q', + '-d', $pair->node1->connstr('postgres'), + '-c', 'BEGIN ISOLATION LEVEL REPEATABLE READ; ' + . 'SELECT 1; ' + . 'SELECT pg_sleep(15); ' + . 'COMMIT;' + ], + \$h_in, \$h_out, \$h_err); + +# Give the holder time to take its snapshot and node1 time to publish a +# conservative (non-sentinel) report at the 2s lmon cadence. +usleep(4_000_000); + +my $floor_hold_a = floor_local($pair->node0); +write_retry($pair->node0, "UPDATE $tbl SET v = v + 1 WHERE k <= 8", 6); +usleep(2_500_000); +my $floor_hold_b = floor_local($pair->node0); + +is($floor_hold_b, $floor_hold_a, + 'L7 HARD ASSERT: floor clamped while node1 held a live snapshot'); + +$holder->finish; +is($h_err, '', 'L7d holder session ran clean (snapshot actually held)'); +my $sentinel_post_hold = + state_val($pair->node1, 'undo', 'horizon_idle_sentinel_sent_count'); + +# The hold spanned ~7 report ticks; a sentinel-per-tick would have advanced +# the gauge by ~7. At most one pre-snapshot tick may race in. +cmp_ok($sentinel_post_hold - $sentinel_pre_hold, '<=', 1, + 'L7c sentinel suppressed while node1 held a live snapshot'); + +write_retry($pair->node0, "UPDATE $tbl SET v = v + 1 WHERE k = 1", 2); +ok(wait_for(sub { floor_local($pair->node0) > $floor_hold_b }, 20), + 'L7b floor resumed after the holder released'); + +# ============================================================ +# L8: wake-up self-heal — node1's first cross-node reads after the +# storm succeed. A bounded number of retries is allowed: a snapshot +# taken below a just-recycled bound refuses fail-closed +# (BELOW_HORIZON inadmissible) and the wire observe heals the next +# snapshot. Quiesce the table first (owner-side VACUUM, t/394 +# recipe) so low-xid "TT slot recycled" cleanout noise cannot +# masquerade as the mechanism under test. +# ============================================================ +# Owner-side cleanout MUST succeed (t/394 recipe: single-writer table, so +# FREEZE is safe): recycled-slot ITL refs left on the heap would otherwise +# fail-close every node1 read with "TT slot recycled" -- the documented +# low-xid artifact, not the mechanism under test. +# +# The cleanout needs node1-mastered GES resources (VM fork / relation +# locks) and the storm can leave node1's master work queue saturated for +# minutes (source=master-reject-queue-full at elapsed_ms=0 -- the S3 +# lock-timeout wedge class; structural fix owned by the convert-queue +# lane). So: try in place first; if the plane stays wedged, bounce node1 +# -- the same operational recovery the S3 rig used -- and retry. Once the +# queue-lane fix lands, the in-place path takes over by itself. Either +# way the leg HARD-fails if cleanout never succeeds: the wedge is +# surfaced, not papered over. +my $vac_ok = 0; +for my $i (1 .. 6) +{ + my ($rc, $out, $err) = $pair->node0->psql('postgres', "VACUUM (FREEZE) $tbl"); + if ($rc == 0) { $vac_ok = 1; last; } + usleep(2_500_000); +} +if (!$vac_ok) +{ + diag('in-place cleanout blocked (post-storm master-queue saturation); ' + . 'bouncing node1'); + $pair->node1->stop('fast'); + $pair->node1->start; + ok($pair->wait_for_peer_state(0, 1, 'connected', 60) + && $pair->wait_for_peer_state(1, 0, 'connected', 60), + 'L8a0 node1 bounced and reconnected'); + for my $i (1 .. 6) + { + my ($rc, $out, $err) = $pair->node0->psql('postgres', "VACUUM (FREEZE) $tbl"); + if ($rc == 0) { $vac_ok = 1; last; } + usleep(2_500_000); + } + + # Flush the frozen pages to the shared file: a REJOINED node + # silently reads the on-disk pages of a table whose current + # versions live only in the peer's buffers -- a PRE-EXISTING + # silent-empty-read defect reproduced minimally on main WITHOUT + # this test's storm (probe archived in the lane's evidence bundle; + # reported separately, owned by the GCS read path). The + # checkpoint keeps THIS leg about the floor/wake semantics instead + # of that defect; drop it once that fix lands. + my $ckpt_ok = 0; + for my $i (1 .. 6) + { + my ($rc, $out, $err) = $pair->node0->psql('postgres', 'CHECKPOINT'); + if ($rc == 0) { $ckpt_ok = 1; last; } + usleep(2_500_000); + } + ok($ckpt_ok, 'L8a-post node0 checkpoint after cleanout (pre-fix rejoin-read workaround)'); +} +else +{ + ok(1, 'L8a0 in-place cleanout path (no node1 bounce needed)'); +} +ok($vac_ok, 'L8a owner-side cleanout (VACUUM FREEZE) succeeded'); + +my ($sum, $tries) = ('', 0); +for my $i (1 .. 10) +{ + $tries = $i; + my ($rc, $stdout, $stderr) = + $pair->node1->psql('postgres', "SELECT count(*), sum(v) >= 0 FROM $tbl"); + if ($rc == 0 && $stdout =~ /^64\|t$/) + { + $sum = $stdout; + last; + } + diag(sprintf('L8 attempt %d: rc=%d out=[%s] err=[%s]', + $i, $rc, $stdout // '', join(' / ', split(/\n/, $stderr // '')))); + usleep(1_000_000); +} +is($sum, '64|t', "L8 node1 cross-node read healed (tries=$tries)"); + +# ============================================================ +# L9: restart clean. +# ============================================================ +$pair->node0->stop('fast'); +$pair->node1->stop('fast'); +$pair->node0->start; +$pair->node1->start; +usleep(3_000_000); +ok($pair->wait_for_peer_state(0, 1, 'connected', 60) + && $pair->wait_for_peer_state(1, 0, 'connected', 60), + 'L9 cluster restarted after both legs'); +my $restart_count = ''; +for my $i (1 .. 5) +{ + my ($rc, $stdout, $stderr) = + $pair->node0->psql('postgres', "SELECT count(*) FROM $tbl"); + if ($rc == 0) { $restart_count = $stdout; last; } + usleep(1_000_000); +} +is($restart_count, '64', 'L9 table readable after restart'); + +done_testing(); diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index 1059c08a28..8f1964089f 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -2740,6 +2740,11 @@ cluster_undo_horizon_admission_refuse_count(void) { return 0; } +uint64 +cluster_undo_horizon_idle_sentinel_sent_count(void) +{ + return 0; +} SCN cluster_undo_horizon_last_floor(void) { diff --git a/src/test/cluster_unit/test_cluster_ic.c b/src/test/cluster_unit/test_cluster_ic.c index 192e3649b1..d7efadaea6 100644 --- a/src/test/cluster_unit/test_cluster_ic.c +++ b/src/test/cluster_unit/test_cluster_ic.c @@ -596,9 +596,10 @@ UT_TEST(test_hello_wire_reference_bytes) * CAPS_REPLY_V1 meta bit (0x8) + GCS-race round-2 F6 completion-proof * (0x10) + round-3 P0-1 xid wrap barrier (0x20) + round-4 P0-1 * authority flock (0x40) + ownership-gen ruling② invalidate BUSY - * (0x80) (smart-fusion is off in this fixture) */ + * (0x80) + TT-lane undo-horizon idle sentinel (0x100) + * (smart-fusion is off in this fixture) */ UT_ASSERT_EQ(wire[36], 0xFE); - UT_ASSERT_EQ(wire[37], 0x00); + UT_ASSERT_EQ(wire[37], 0x01); UT_ASSERT_EQ(wire[38], 0x00); UT_ASSERT_EQ(wire[39], 0x00); /* remaining _pad must be all zero: a CONTROL-plane HELLO with @@ -691,12 +692,12 @@ UT_TEST(test_hello_smart_fusion_capability_gate) cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, "sf-off", CLUSTER_IC_PLANE_CONTROL, 0); UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); - UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), - PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1 - | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1 | PGRAC_IC_HELLO_CAP_GCS_DONE_V1 - | PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1 - | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 - | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1); + UT_ASSERT_EQ( + cluster_ic_hello_capabilities(&parsed), + PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1 + | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1 | PGRAC_IC_HELLO_CAP_GCS_DONE_V1 + | PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1 | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 + | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1); cluster_smart_fusion = true; cluster_interconnect_tier = CLUSTER_IC_TIER_2; @@ -704,12 +705,12 @@ UT_TEST(test_hello_smart_fusion_capability_gate) cluster_ic_build_hello(wire, PGRAC_IC_HELLO_VERSION_V1, PGRAC_IC_ENVELOPE_VERSION_V1, 1, "sf-tier-mismatch", CLUSTER_IC_PLANE_CONTROL, 0); UT_ASSERT(cluster_ic_parse_hello(wire, &parsed)); - UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), - PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1 - | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1 | PGRAC_IC_HELLO_CAP_GCS_DONE_V1 - | PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1 - | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 - | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1); + UT_ASSERT_EQ( + cluster_ic_hello_capabilities(&parsed), + PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1 + | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1 | PGRAC_IC_HELLO_CAP_GCS_DONE_V1 + | PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1 | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 + | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1); cluster_smart_fusion = true; cluster_interconnect_tier = CLUSTER_IC_TIER_3; @@ -722,7 +723,8 @@ UT_TEST(test_hello_smart_fusion_capability_gate) PGRAC_IC_HELLO_CAP_SMART_FUSION_REPLY_V2 | PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1 | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1 | PGRAC_IC_HELLO_CAP_GCS_DONE_V1 | PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1 - | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1); + | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 + | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1); cluster_smart_fusion = false; cluster_interconnect_tier = CLUSTER_IC_TIER_STUB; diff --git a/src/test/cluster_unit/test_cluster_undo_horizon.c b/src/test/cluster_unit/test_cluster_undo_horizon.c index 7641fa9a1c..63860ab0a1 100644 --- a/src/test/cluster_unit/test_cluster_undo_horizon.c +++ b/src/test/cluster_unit/test_cluster_undo_horizon.c @@ -451,6 +451,61 @@ UT_TEST(test_u16b_blame_order) UT_ASSERT_EQ(r.blame, 1); } +/* ---- U17: idle-unconstrained sentinel never wins the min fold -------- */ +UT_TEST(test_u17_idle_sentinel_skipped) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[3]; + FoldResult r; + + views[0] = mk_view(0); /* self slot: ignored */ + views[1] = mk_view((uint64)CLUSTER_UNDO_HORIZON_REPORT_UNCONSTRAINED); + views[2] = mk_view(900); + bm_set(req, 1); + bm_set(req, 2); + + /* the sentinel peer contributes nothing; min(local 800, peer2 900) */ + r = run_fold((SCN)800, views, 3, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_OK); + UT_ASSERT_EQ((uint64)r.floor.scn, (uint64)800); + UT_ASSERT_EQ(r.floor.epoch, TEST_EPOCH); + + /* every required peer idle => floor is exactly the local horizon */ + views[2] = mk_view((uint64)CLUSTER_UNDO_HORIZON_REPORT_UNCONSTRAINED); + r = run_fold((SCN)800, views, 3, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_OK); + UT_ASSERT_EQ((uint64)r.floor.scn, (uint64)800); +} + +/* ---- U17b: sentinel reports still owe every proof obligation --------- */ +UT_TEST(test_u17b_idle_sentinel_still_proven) +{ + uint8 req[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + ClusterUndoHorizonReportView views[2]; + FoldResult r; + + /* a STALE sentinel stalls exactly like a stale real report: an idle + * peer that stops reporting is unproven coverage, not "unconstrained + * forever" */ + views[0] = mk_view(0); + views[1] = mk_view((uint64)CLUSTER_UNDO_HORIZON_REPORT_UNCONSTRAINED); + views[1].recv_at_us = TEST_NOW_US - (uint64)10 * 1000 * 1000; /* 10s old */ + bm_set(req, 1); + + r = run_fold((SCN)800, views, 2, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_STALLED); + UT_ASSERT_EQ(r.reason, CLUSTER_UNDO_HORIZON_STALL_STALE); + UT_ASSERT_EQ(r.blame, 1); + + /* an epoch-mismatched sentinel stalls too */ + views[1] = mk_view((uint64)CLUSTER_UNDO_HORIZON_REPORT_UNCONSTRAINED); + views[1].epoch = TEST_EPOCH - 1; + r = run_fold((SCN)800, views, 2, req); + UT_ASSERT_EQ(r.st, CLUSTER_UNDO_HORIZON_FOLD_STALLED); + UT_ASSERT_EQ(r.reason, CLUSTER_UNDO_HORIZON_STALL_EPOCH); + UT_ASSERT_EQ(r.blame, 1); +} + /* ---- reason names cover every enum arm ------------------------------ */ UT_TEST(test_reason_names) { @@ -475,7 +530,7 @@ UT_TEST(test_reason_names) int main(void) { - UT_PLAN(19); + UT_PLAN(21); UT_RUN(test_u1_no_required_peer); UT_RUN(test_u2_all_fresh_min); @@ -495,6 +550,8 @@ main(void) UT_RUN(test_u14_regression); UT_RUN(test_u16_nocap); UT_RUN(test_u16b_blame_order); + UT_RUN(test_u17_idle_sentinel_skipped); + UT_RUN(test_u17b_idle_sentinel_still_proven); UT_RUN(test_reason_names); UT_DONE(); From 10663c5bb443fef3bb37fecc85b0bf83afc96d45 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 15 Jul 2026 19:08:09 +0800 Subject: [PATCH 2/2] test(cluster): pin the gated rtvis self-heal as a RED->GREEN twin (t/403 L10) One driver, no cleanout bypass: a node0 xid whose TT slot is provably lapped (>=2 pool rollovers, hard-floor assert) is read from node1 twice. RED twin under the default cluster.crossnode_runtime_visibility = off: the read MUST fail closed 53R97 ("cluster TT status unknown") with node1's rtvis_verdict_wire_count provably flat -- a succeeding read is a hard fail, never retried, so the GREEN result cannot come from an overlay / hint / delayed-cleanout side door. GREEN twin after flipping ONLY the GUC (PGC_SUSET reload, both nodes): the same read returns the exact terminal 64|2080 and node1's verdict counters move (wire=64, below_horizon=64) -- every row resolved through the origin-verdict COMMITTED_BELOW_HORIZON arm (cluster_runtime_visibility.c). cluster.tt_status_hint_emit_mode is disabled (PGC_SIGHUP) before the fixture write so the V4 hint wire cannot install the terminal status into node1's TT overlay at commit time: this deterministically manufactures the "hint never arrived" state the resolve path exists for (the outbound hint ring drops under storm load anyway) -- it forces the mechanism under test, it does not weaken any assertion. Known structural limit (registered follow-up, not this leg): with cluster.xid_striping off (this rig) classify_ref's recycled-ref derivation branch is unreachable -- only the fresh-ref AUTHORITATIVE ask is served; a striping-on variant leg needs a PGC_POSTMASTER re-provision. Gates: t/403 33/33 x3 independent runs; L1-L9 assertions unweakened; test-only diff. Spec: spec-5.22e-undo-cluster-retention-horizon.md --- .../t/403_undo_idle_peer_floor_pin_2node.pl | 240 ++++++++++++++++++ 1 file changed, 240 insertions(+) diff --git a/src/test/cluster_tap/t/403_undo_idle_peer_floor_pin_2node.pl b/src/test/cluster_tap/t/403_undo_idle_peer_floor_pin_2node.pl index fabe9db532..a40e8d48a0 100644 --- a/src/test/cluster_tap/t/403_undo_idle_peer_floor_pin_2node.pl +++ b/src/test/cluster_tap/t/403_undo_idle_peer_floor_pin_2node.pl @@ -53,6 +53,14 @@ # L8 wake-up self-heal: node1's first cross-node read after the # storm succeeds (bounded retries allowed: the inadmissible # BELOW_HORIZON arm is fail-closed and observe heals it) +# L10 GATED self-heal proof (runs before L9), RED->GREEN twin +# on ONE driver with NO cleanout bypass: a recycled node0 +# xid first fails closed 53R97 under the default +# cluster.crossnode_runtime_visibility = off (verdict wire +# provably untouched), then -- GUC flipped on, nothing else +# changed -- resolves through the COMMITTED_BELOW_HORIZON +# origin-verdict arm: exact data terminal + rtvis verdict +# counter evidence # L9 restart clean # # Spec: spec-5.22e (cluster undo retention brake) S2.1 sampling rule + @@ -500,6 +508,238 @@ sub floor_lag } is($sum, '64|t', "L8 node1 cross-node read healed (tries=$tries)"); +# ============================================================ +# L10: the GATED wake-up self-heal, for real -- the +# COMMITTED_BELOW_HORIZON resolve arm under +# cluster.crossnode_runtime_visibility = on, with NO owner-side +# cleanout bypass (no FREEZE, no CHECKPOINT). +# +# L8 above only proves the read heals AFTER a VACUUM (FREEZE) +# removed the remote ITL refs -- the rtvis resolve machinery +# (cluster_runtime_visibility.c rtvis_try_origin_verdict) never ran +# there because the GUC defaults off. This leg drives it as a +# RED->GREEN twin on ONE driver: +# +# 1. cluster.tt_status_hint_emit_mode = disabled (PGC_SIGHUP, +# default all_status) BEFORE the fixture write: otherwise the +# V4 hint wire installs the fixture xid's terminal status into +# node1's TT overlay at commit time and the read resolves from +# the overlay without ever touching the verdict wire. This is +# NOT an assertion-weakening bypass: it deterministically +# manufactures the "hint never arrived" state the resolve path +# exists for (the 256-slot outbound ring drops hints under +# storm load anyway), FORCING the mechanism under test to carry +# the read. cluster.crossnode_runtime_visibility stays at its +# default (off) until the RED twin below has run. +# 2. Fixture: ONE node0 transaction inserts 64 rows -> a single +# fresh remote ITL ref (local_xid == raw_xid). With +# cluster.xid_striping off in this rig, cluster_xid_is_mine() +# is always false at the origin, so ONLY the spec-5.22f +# fresh-ref AUTHORITATIVE ask is ever served -- the fixture must +# stay fresh (single writer xact, page never revisited by +# node0, so no delayed cleanout stamps it). +# 3. Churn: a node0 writer laps the TT pool (same +# tt_retention_rollover_count evidence as L2), so the fixture +# xid's TT slot is provably rebound; the origin's complete +# by-xid scan then 0-matches and -- retention legs (a)-(d) + +# CLOG committed -- serves COMMITTED_BELOW_HORIZON{H}. +# 4. RED twin: with the GUC still off, node1's read of the fixture +# MUST fail closed 53R97 ("cluster TT status unknown", +# heapam_visibility.c) and node1's verdict-wire counter MUST +# stay flat (cluster_runtime_visibility.c gates the whole +# resolve on the GUC before any wire touch). This proves the +# recycled-slot window is genuinely open at the moment the +# GREEN twin runs -- the GREEN result cannot be an overlay / +# hint / cleanout side door. +# 5. cluster.crossnode_runtime_visibility = on (PGC_SUSET, reload) +# on BOTH nodes: the requester's classify / fresh-ref widening +# gates on it AND the origin's lms_undo_verdict_serve refuses +# while it is off (cluster_cr_server.c). +# 6. GREEN twin: node1 re-reads the same fixture. A snapshot +# read_scn behind the shipped horizon takes the inadmissible +# fail-closed arm and the Lamport observe heals the NEXT +# snapshot (bounded retries, the documented self-heal); the +# healed snapshot admits the bound and every row resolves +# visible. +# +# HARD ASSERTS: RED twin fails closed with the wire flat + exact +# terminal data on the GREEN twin (count=64 AND sum=2080 -- not +# L8's sum >= 0 shape) + node1's rtvis verdict counters moved +# (below_horizon / exact), proving the verdict wire resolved the +# read rather than an overlay / hint / cleanout side door. +# +# Known hazard (documented, not worked around): if L8 had to bounce +# node1, the pre-existing rejoin silent-empty-read defect (see L8 +# comment) can surface here as count=0 -- that is a REAL defect this +# leg must fail on, not paper over. +# ============================================================ +for my $node ($pair->node0, $pair->node1) +{ + $node->safe_psql('postgres', + "ALTER SYSTEM SET cluster.tt_status_hint_emit_mode = 'disabled'"); + $node->safe_psql('postgres', 'SELECT pg_reload_conf()'); +} +ok( wait_for( + sub { + $pair->node0->safe_psql('postgres', + 'SHOW cluster.tt_status_hint_emit_mode') eq 'disabled' + && $pair->node1->safe_psql('postgres', + 'SHOW cluster.tt_status_hint_emit_mode') eq 'disabled'; + }, + 15), + 'L10a status-hint wire disabled via reload (both nodes)'); + +# Coinciding-filepath fixture on shared storage (same t/394 recipe as +# L1b; no shared catalog in this rig). +my $l10tbl; +for my $i (1 .. 12) +{ + my $t = "t403l10_$i"; + $_->safe_psql('postgres', "CREATE TABLE $t (k int, v int)") + for ($pair->node0, $pair->node1); + my $p0 = $pair->node0->safe_psql('postgres', "SELECT pg_relation_filepath('$t')"); + my $p1 = $pair->node1->safe_psql('postgres', "SELECT pg_relation_filepath('$t')"); + if (($p0 // '') eq ($p1 // '')) { $l10tbl = $t; last; } +} +ok(defined $l10tbl, 'L10b coinciding-filepath fixture table found'); +die 'no coinciding filepath found for L10' unless defined $l10tbl; +diag("L10 fixture table=$l10tbl"); + +# ONE transaction writes all 64 rows: exact expected terminal data is +# count=64, sum(v)=sum(1..64)=2080, and the page ITL stays a single +# fresh entry bound to this xid. +$pair->node0->safe_psql('postgres', + "INSERT INTO $l10tbl SELECT g, g FROM generate_series(1, 64) g"); + +# Churn phase: lap the TT pool on node0 so the fixture xid's slot is +# rebound (>= 2 laps hard floor, target 3; single background writer on +# a node0-only table so the fixture pages are never revisited). +$pair->node0->safe_psql('postgres', 'CREATE TABLE t403l10_churn (k int)'); +my $churnfile = "$tmpdir/churn.sql"; +{ + open(my $fh, '>', $churnfile) or die "cannot write $churnfile: $!"; + print $fh "INSERT INTO t403l10_churn VALUES (1);\n" for (1 .. $WRITER_LINES); + close($fh); +} +my $roll_pre_l10 = state_val($pair->node0, 'undo', 'tt_retention_rollover_count'); +my ($c_in, $c_out, $c_err) = ('', '', ''); +my $churn = IPC::Run::start( + [ + 'psql', '-X', '-A', '-t', '-q', + '-d', $pair->node0->connstr('postgres'), + '-f', $churnfile + ], + \$c_in, \$c_out, \$c_err); +my $churn_t0 = time(); +my $roll_delta_l10 = 0; +while (time() - $churn_t0 < 120) +{ + $roll_delta_l10 = + state_val($pair->node0, 'undo', 'tt_retention_rollover_count') - $roll_pre_l10; + last if $roll_delta_l10 >= 3 * $POOL_SEGMENTS; + usleep(400_000); +} +$churn->kill_kill; +diag(sprintf('L10 churn: %d TT rollovers in %.0fs (hard floor %d)', + $roll_delta_l10, time() - $churn_t0, 2 * $POOL_SEGMENTS)); +cmp_ok($roll_delta_l10, '>=', 2 * $POOL_SEGMENTS, + 'L10c churn lapped the TT pool (fixture slot provably rebound)'); + +# ------------------------------------------------------------ +# RED twin: cluster.crossnode_runtime_visibility is still at its +# default (off). The same read the GREEN twin will heal below MUST +# fail closed here -- 53R97 "cluster TT status unknown" +# (heapam_visibility.c) -- and node1's verdict-wire counter MUST stay +# flat: cluster_runtime_visibility.c refuses before any wire touch +# while the GUC is off, so a moving counter (or a succeeding read) +# would mean a side door carried it. Transient non-53R97 errors +# (post-churn GES tail, the L8 class) are retried; a SUCCEEDING read +# is a hard fail, never retried. +# ------------------------------------------------------------ +my $wire_pre_neg = state_val($pair->node1, 'cr', 'rtvis_verdict_wire_count'); +my ($neg_hit_53r97, $neg_unexpected) = (0, ''); +for my $i (1 .. 10) +{ + my ($rc, $stdout, $stderr) = + $pair->node1->psql('postgres', "SELECT count(*), sum(v) FROM $l10tbl"); + if ($rc == 0) + { + $neg_unexpected = 'read SUCCEEDED with rtvis off: [' . ($stdout // '') . ']'; + last; + } + if (($stderr // '') =~ /cluster TT status unknown for xid/) + { + $neg_hit_53r97 = 1; + last; + } + $neg_unexpected = join(' / ', split(/\n/, $stderr // '')); + diag(sprintf('L10d RED-twin attempt %d: rc=%d non-53R97 err=[%s]', + $i, $rc, $neg_unexpected)); + usleep(1_500_000); +} +ok($neg_hit_53r97, + 'L10d RED twin HARD ASSERT: rtvis off fails the read closed 53R97 ' + . '(cluster TT status unknown)') + or diag("L10d terminal state: $neg_unexpected"); +is(state_val($pair->node1, 'cr', 'rtvis_verdict_wire_count') - $wire_pre_neg, + 0, 'L10e RED twin: verdict wire provably untouched while the GUC is off'); + +# GREEN twin arming: flip ONLY the GUC; driver and fixture unchanged. +for my $node ($pair->node0, $pair->node1) +{ + $node->safe_psql('postgres', + 'ALTER SYSTEM SET cluster.crossnode_runtime_visibility = on'); + $node->safe_psql('postgres', 'SELECT pg_reload_conf()'); +} +ok( wait_for( + sub { + $pair->node0->safe_psql('postgres', + 'SHOW cluster.crossnode_runtime_visibility') eq 'on' + && $pair->node1->safe_psql('postgres', + 'SHOW cluster.crossnode_runtime_visibility') eq 'on'; + }, + 15), + 'L10f crossnode_runtime_visibility on via reload (both nodes)'); + +# Requester-side rtvis verdict counters live on node1 (category 'cr'). +my $bh_pre = state_val($pair->node1, 'cr', 'rtvis_verdict_below_horizon_count'); +my $ex_pre = state_val($pair->node1, 'cr', 'rtvis_verdict_exact_count'); +my $inadm_pre = state_val($pair->node1, 'cr', 'rtvis_verdict_inadmissible_count'); +my $wire_pre = state_val($pair->node1, 'cr', 'rtvis_verdict_wire_count'); + +# node1 cross-node read, bounded retries: the inadmissible +# BELOW_HORIZON arm is fail-closed and the wire observe heals the next +# snapshot (plus post-churn GES tail tolerance, as in L8). +my ($l10_out, $l10_tries) = ('', 0); +for my $i (1 .. 15) +{ + $l10_tries = $i; + my ($rc, $stdout, $stderr) = + $pair->node1->psql('postgres', "SELECT count(*), sum(v) FROM $l10tbl"); + if ($rc == 0 && $stdout =~ /^64\|2080$/) + { + $l10_out = $stdout; + last; + } + diag(sprintf('L10 attempt %d: rc=%d out=[%s] err=[%s]', + $i, $rc, $stdout // '', join(' / ', split(/\n/, $stderr // '')))); + usleep(1_500_000); +} +is($l10_out, '64|2080', + "L10 HARD ASSERT: node1 healed read returned exact terminal data (tries=$l10_tries)"); + +my $bh_d = state_val($pair->node1, 'cr', 'rtvis_verdict_below_horizon_count') - $bh_pre; +my $ex_d = state_val($pair->node1, 'cr', 'rtvis_verdict_exact_count') - $ex_pre; +my $inadm_d = state_val($pair->node1, 'cr', 'rtvis_verdict_inadmissible_count') - $inadm_pre; +my $wire_d = state_val($pair->node1, 'cr', 'rtvis_verdict_wire_count') - $wire_pre; +diag("L10 node1 rtvis verdict deltas: wire=$wire_d below_horizon=$bh_d " + . "exact=$ex_d inadmissible=$inadm_d"); +cmp_ok($bh_d + $ex_d, '>', 0, + 'L10g HARD ASSERT: verdict resolve path really carried the read ' + . '(below_horizon + exact advanced)'); +cmp_ok($bh_d, '>', 0, + 'L10h below-horizon arm served the recycled fixture xid'); + # ============================================================ # L9: restart clean. # ============================================================