fix(undo): unpin the idle-peer recycle floor and let the cleaner keep pace

.github/workflows/nightly.yml

@@ -239,6 +239,14 @@ jobs:
           # Own shard: six ClusterPair bring-ups need the wall clock
           # (L342: every new t/ file lands in a shard the same commit).
           - { name: stage7-gcs-ownership-gen, ranges: "392-397", unit: false, regress: false }
+          # t/401 GCS master-direct STALE-ship storage rescue (PR#55) +
+          # t/402 undo cleaner stall-retry cadence (PR#56) + t/403 TT-lane
+          # idle-peer floor pin under write load (all 2-node pairs from the
+          # S3 wave).  t/401/402 predate this shard and were in NO CI
+          # matrix -- closes that hole (L342: every new t/ file lands in a
+          # shard the same commit).  Own shard: three ClusterPair bring-ups
+          # + the t/403 sustained write phase need the wall clock.
+          - { name: stage7-s3-wave, ranges: "401-403", unit: false, regress: false }
     steps:
       - name: Checkout
         uses: actions/checkout@v4

src/backend/cluster/cluster_debug.c

@@ -2877,6 +2877,8 @@ dump_undo(ReturnSetInfo *rsinfo)
 			 fmt_int64((int64)cluster_undo_horizon_admission_refuse_count()));
 	emit_row(rsinfo, "undo", "horizon_last_floor_scn",
 			 fmt_int64((int64)cluster_undo_horizon_last_floor()));
+	emit_row(rsinfo, "undo", "horizon_idle_sentinel_sent_count",
+			 fmt_int64((int64)cluster_undo_horizon_idle_sentinel_sent_count()));
 	emit_row(rsinfo, "undo", "horizon_peer_reports", cluster_undo_horizon_peer_reports_summary());
 	emit_row(rsinfo, "undo", "cleaner_header_tt_slots_below_horizon",
 			 fmt_int64((int64)cluster_undo_cleaner_header_tt_slots_below_horizon()));

src/backend/cluster/cluster_ic.c

@@ -656,6 +656,10 @@ cluster_ic_build_hello(uint8 out_buf[PGRAC_IC_HELLO_BYTES], uint16 hello_version
 	 * negative-ACK protocol capability, same unconditional discipline (see
 	 * cluster_ic.h). */
 	capabilities |= PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1;
+	/* PGRAC TT lane (S3 idle-peer floor pin): undo-horizon idle-unconstrained
+	 * sentinel protocol capability, same unconditional discipline (see
+	 * cluster_ic.h). */
+	capabilities |= PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1;
 	if (capabilities != 0)
 		ic_le_write_uint32(out_buf + PGRAC_IC_HELLO_CAPABILITIES_OFFSET, capabilities);
 

src/backend/cluster/cluster_sf_dep.c

@@ -235,6 +235,30 @@ cluster_sf_peer_supports_undo_horizon(int32 peer_id)
 	return (capabilities & PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1) != 0;
 }
 
+/*
+ * cluster_sf_peer_supports_undo_horizon_idle
+ *
+ * TT lane (S3 idle-peer floor pin) capability gate: true iff the peer's
+ * verified HELLO on the CURRENT connection advertised the idle-unconstrained
+ * sentinel report bit.  Same connection-bound, no-local-GUC discipline as
+ * the queries above.  Consumed by the SENDER only: a provably idle node
+ * sends the sentinel exclusively to peers that understand it; an old peer
+ * keeps receiving the conservative clock sample (see cluster_ic.h).
+ */
+bool
+cluster_sf_peer_supports_undo_horizon_idle(int32 peer_id)
+{
+	uint32 capabilities;
+
+	if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES)
+		return false;
+
+	LWLockAcquire(&ClusterSfDep->lock, LW_SHARED);
+	capabilities = cluster_sf_peer_cap_bits(&ClusterSfDep->peer_capabilities[peer_id]);
+	LWLockRelease(&ClusterSfDep->lock);
+	return (capabilities & PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1) != 0;
+}
+
 /*
  * cluster_sf_peer_supports_gcs_done
  *

src/backend/cluster/cluster_undo_cleaner.c

@@ -364,13 +364,24 @@ undo_cleaner_advance_liveness_tick(void)
  *	cluster-wide COMMITTED recycling for 30-60s).  Rule 8.A unchanged:
  *	a retry still recycles ONLY on a proven floor; a persistent cause
  *	keeps every retry stalled.
+ *
+ *	*out_work_remaining (TT lane, S3 idle-peer floor pin H2): true when
+ *	this pass BOTH consumed its whole segment batch AND made recycle
+ *	progress — the backlog is bigger than one batch, so the caller
+ *	re-runs immediately (pressure-driven continuous mode) instead of
+ *	sleeping the recycle interval.  Pure cadence: what may be recycled
+ *	is still decided only by the proven floor above.  A pinned pass
+ *	(zero progress) or a drained backlog (batch not exhausted) reports
+ *	false, so the continuous mode can never busy-spin.
  */
 static bool
-undo_cleaner_run_pass(void)
+undo_cleaner_run_pass(bool *out_work_remaining)
 {
 	ClusterUndoCleanerPassStats stats;
 	bool floor_retry_needed = false;
 
+	*out_work_remaining = false;
+
 	if (!cluster_undo_cleaner_enabled)
 		return false;
 	if (!cluster_storage_mode_enabled())
@@ -540,6 +551,17 @@ undo_cleaner_run_pass(void)
 			if (fence_aborted) {
 				floor_retry_needed = true;
 				cluster_undo_horizon_note_pass_abort();
+			} else if (batch == 0
+					   && (stats.segments_marked_recyclable > 0 || stats.shmem_tt_slots_gcd > 0
+						   || stats.header_tt_slots_below_horizon > 0)) {
+				/*
+				 * H2 continuous mode: the batch budget ran out while
+				 * recycle progress was still being made -- more eligible
+				 * inventory is waiting than one batch covers.  Ask the
+				 * caller to re-run now rather than let a write storm
+				 * outrun the interval cadence.
+				 */
+				*out_work_remaining = true;
 			}
 		}
 	}
@@ -634,6 +656,7 @@ UndoCleanerMain(void)
 		int rc;
 		int timeout_ms;
 		bool floor_retry;
+		bool work_remaining;
 
 		CHECK_FOR_INTERRUPTS();
 
@@ -649,7 +672,20 @@ UndoCleanerMain(void)
 
 		CLUSTER_INJECTION_POINT("undo-cleaner-main-loop-iter");
 
-		floor_retry = undo_cleaner_run_pass();
+		floor_retry = undo_cleaner_run_pass(&work_remaining);
+
+		/*
+		 * TT lane H2 (pressure-driven continuous mode): a pass that
+		 * exhausted its segment batch while still recycling has a backlog
+		 * bigger than one batch -- under a write storm the interval
+		 * cadence is orders of magnitude too slow (12k xacts/s consume a
+		 * 48-slot segment every ~4ms), so loop straight into the next
+		 * pass.  The loop head re-checks interrupts / reload / shutdown,
+		 * and the signal requires PROGRESS, so a pinned or stalled floor
+		 * always falls through to the waits below.
+		 */
+		if (work_remaining)
+			continue;
 
 		/*
 		 * interval 0 = pressure-wakeup only (Q8): block without

src/backend/cluster/cluster_undo_horizon_ic.c

@@ -91,8 +91,9 @@ typedef struct ClusterUndoHorizonShmem {
 	pg_atomic_uint64 pass_abort_count;
 	pg_atomic_uint64 wire_reject_count;
 	pg_atomic_uint64 admission_refuse_count;
-	pg_atomic_uint64 last_floor_scn;	  /* gauge: last OK fold result */
-	pg_atomic_uint64 self_admitted_epoch; /* D5-8: (epoch self last became
+	pg_atomic_uint64 idle_sentinel_sent_count; /* TT lane: sentinel sends */
+	pg_atomic_uint64 last_floor_scn;		   /* gauge: last OK fold result */
+	pg_atomic_uint64 self_admitted_epoch;	   /* D5-8: (epoch self last became
 										   * MEMBER) + 1; 0 = never admitted.
 										   * Biased so a cold-formation
 										   * admission at epoch 0 is distinct
@@ -126,6 +127,7 @@ cluster_undo_horizon_shmem_init(void)
 		pg_atomic_init_u64(&shmem->pass_abort_count, 0);
 		pg_atomic_init_u64(&shmem->wire_reject_count, 0);
 		pg_atomic_init_u64(&shmem->admission_refuse_count, 0);
+		pg_atomic_init_u64(&shmem->idle_sentinel_sent_count, 0);
 		pg_atomic_init_u64(&shmem->last_floor_scn, 0);
 		pg_atomic_init_u64(&shmem->self_admitted_epoch, 0);
 	}
@@ -239,18 +241,33 @@ cluster_undo_horizon_report_handler(const ClusterICEnvelope *env, const void *pa
 	 * snapshot BELOW the accepted bound exists over there -- latch the
 	 * violation (the fold stalls, U14) and do not move the payload.  A
 	 * later conforming report clears the latch.
+	 *
+	 * The idle-unconstrained sentinel (TT lane) is OUTSIDE the monotone
+	 * sequence: it claims "no constraint", not a bound, so it neither
+	 * regresses against the accepted base (its scn_local is the ceiling
+	 * anyway) nor BECOMES the base -- otherwise the sender's first real
+	 * report after waking (its clock, numerically below the sentinel)
+	 * would latch a spurious regression and stall the fold on every
+	 * idle->active transition.  A real report after a sentinel still
+	 * compares against the last REAL accepted value (the sender's clock
+	 * is monotone, so that comparison holds across the idle gap).
 	 */
-	if (slot->accepted_any && slot->accepted_epoch == wire.epoch
-		&& scn_time_cmp((SCN)wire.horizon_scn, (SCN)slot->accepted_scn) < 0) {
-		undo_horizon_reject("same-epoch regression", sender);
-		undo_horizon_slot_publish(slot, &wire, now_us, true);
+	if ((SCN)wire.horizon_scn != CLUSTER_UNDO_HORIZON_REPORT_UNCONSTRAINED) {
+		if (slot->accepted_any && slot->accepted_epoch == wire.epoch
+			&& scn_time_cmp((SCN)wire.horizon_scn, (SCN)slot->accepted_scn) < 0) {
+			undo_horizon_reject("same-epoch regression", sender);
+			undo_horizon_slot_publish(slot, &wire, now_us, true);
+			return;
+		}
+
+		undo_horizon_slot_publish(slot, &wire, now_us, false);
+		slot->accepted_scn = wire.horizon_scn;
+		slot->accepted_epoch = wire.epoch;
+		slot->accepted_any = true;
 		return;
 	}
 
 	undo_horizon_slot_publish(slot, &wire, now_us, false);
-	slot->accepted_scn = wire.horizon_scn;
-	slot->accepted_epoch = wire.epoch;
-	slot->accepted_any = true;
 }
 
 void
@@ -305,6 +322,7 @@ cluster_undo_horizon_lmon_tick(void)
 	ClusterUndoHorizonWire wire;
 	uint64 now_us;
 	SCN report;
+	bool unconstrained;
 	int pi;
 
 	if (!cluster_enabled || cluster_node_id < 0 || UndoHorizonShmem == NULL)
@@ -320,11 +338,25 @@ cluster_undo_horizon_lmon_tick(void)
 		return;
 	last_sent_us = now_us;
 
+	/*
+	 * TT lane (S3 idle-peer floor pin): a PROVABLY idle node (zero active
+	 * xacts AND zero held snapshots) reports the unconstrained sentinel so
+	 * its clock-sample lag stops pinning a lone writer's recycle floor.
+	 * MRP-active standbys are never "idle" in this sense (their future
+	 * snapshots read at consistent_scn, S3.0 row 4), and any uncertainty
+	 * in the probe answers false -- the conservative sample then flows
+	 * exactly as before.  The probe runs AFTER the sample: a snapshot
+	 * appearing in between flips it to false (conservative); one
+	 * disappearing in between just means the constraint is gone.
+	 */
+	unconstrained = !cluster_mrp_should_start() && cluster_undo_retention_all_quiescent();
+
 	wire.epoch = cluster_epoch_get_current();
-	wire.horizon_scn = (uint64)report;
 	wire.sender_interval_ms = (uint32)cluster_lmon_main_loop_interval;
 
 	for (pi = 0; pi < CLUSTER_MAX_NODES; pi++) {
+		bool sentinel;
+
 		if (pi == cluster_node_id)
 			continue;
 		if (cluster_conf_lookup_node(pi) == NULL)
@@ -340,6 +372,18 @@ cluster_undo_horizon_lmon_tick(void)
 		if (!cluster_sf_peer_supports_undo_horizon(pi))
 			continue;
 
+		/*
+		 * Sentinel send-side hard gate: an old receiver would fold the
+		 * raw sentinel value harmlessly but latch a spurious same-epoch
+		 * regression when this node wakes, so it keeps getting the
+		 * conservative clock sample instead (see cluster_ic.h).
+		 */
+		sentinel = unconstrained && cluster_sf_peer_supports_undo_horizon_idle(pi);
+		wire.horizon_scn
+			= sentinel ? (uint64)CLUSTER_UNDO_HORIZON_REPORT_UNCONSTRAINED : (uint64)report;
+		if (sentinel)
+			pg_atomic_fetch_add_u64(&UndoHorizonShmem->idle_sentinel_sent_count, 1);
+
 		(void)cluster_ic_send_envelope(PGRAC_IC_MSG_UNDO_HORIZON, pi, &wire, sizeof(wire));
 		/* fire-and-forget (L456): transport retention / errors surface
 		 * through the transport's own paths; a lost report just ages the
@@ -634,6 +678,16 @@ UNDO_HORIZON_NOTE(pass_abort)
 UNDO_HORIZON_NOTE(wire_reject)
 UNDO_HORIZON_NOTE(admission_refuse)
 
+/* TT lane: sender-side sentinel gauge (bumped inline in the tick; only the
+ * reader half of the NOTE pattern is needed). */
+uint64
+cluster_undo_horizon_idle_sentinel_sent_count(void)
+{
+	return UndoHorizonShmem == NULL
+			   ? 0
+			   : pg_atomic_read_u64(&UndoHorizonShmem->idle_sentinel_sent_count);
+}
+
 void
 cluster_undo_horizon_note_floor(SCN scn)
 {

src/backend/storage/ipc/procarray.c

@@ -2264,6 +2264,56 @@ cluster_undo_retention_horizon(void)
 
 	return SCN_VALID(min) ? min : cluster_scn_current();
 }
+
+/*
+ * cluster_undo_retention_all_quiescent -- TT lane (S3 idle-peer floor pin).
+ *
+ * True iff this node provably holds no undo retention constraint at this
+ * instant: no ProcArray entry has an assigned xid, an xmin, or a published
+ * CLUSTER read_scn.  Both predicate legs (zero active transactions AND
+ * zero held snapshots) are required; any doubt answers false and the
+ * horizon-report sender falls back to the conservative clock sample.
+ *
+ * Prepared transactions' dummy PGPROCs sit in the ProcArray with a valid
+ * xid, so an unresolved prepared xact makes this node non-quiescent — the
+ * conservative direction (its undo must stay reachable).  Walsenders /
+ * background workers with an xmin likewise read as constraints.
+ *
+ * Same locking contract as cluster_undo_retention_horizon() above: SHARED
+ * ProcArrayLock, never called under seg->lock / undo lifecycle_lock (C17).
+ * The xid reads follow the ComputeXidHorizons dense-array access pattern.
+ */
+bool
+cluster_undo_retention_all_quiescent(void)
+{
+	ProcArrayStruct *arrayP = procArray;
+	TransactionId *other_xids = ProcGlobal->xids;
+	bool		quiescent = true;
+	int			index;
+
+	if (cluster_node_id < 0)
+		return false;			/* cluster disabled: never claim idle */
+
+	LWLockAcquire(ProcArrayLock, LW_SHARED);
+	for (index = 0; index < arrayP->numProcs; index++)
+	{
+		int			pgprocno = arrayP->pgprocnos[index];
+		PGPROC	   *proc = &allProcs[pgprocno];
+		TransactionId xid = UINT32_ACCESS_ONCE(other_xids[index]);
+		TransactionId xmin = UINT32_ACCESS_ONCE(proc->xmin);
+		SCN			r = (SCN) pg_atomic_read_u64(&proc->cluster_read_scn_atomic);
+
+		if (TransactionIdIsValid(xid) || TransactionIdIsValid(xmin)
+			|| SCN_VALID(r))
+		{
+			quiescent = false;
+			break;
+		}
+	}
+	LWLockRelease(ProcArrayLock);
+
+	return quiescent;
+}
 #endif
 
 /*

src/include/cluster/cluster_ic.h

@@ -350,6 +350,20 @@ typedef enum ClusterICPlane {
  * the pre-BUSY protocol.  Timeout stays the backstop for packet loss / dead
  * nodes. */
 #define PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 ((uint32)0x00000080U)
+/* PGRAC: TT lane (S3 idle-peer floor pin) — this binary understands the
+ * UNDO_HORIZON idle-unconstrained sentinel report (horizon_scn ==
+ * UINT64_MAX): a provably idle sender (zero active xacts AND zero held
+ * snapshots) publishes "no retention constraint" instead of its clock
+ * sample, so a lone writer's recycle floor no longer trails the idle
+ * peers' report cadence (the S3 floor-lag pin: lag x write-rate exceeded
+ * the whole TT slot pool).  A PROTOCOL capability, advertised
+ * unconditionally.  Send-side hard gate: the sentinel is only sent to a
+ * peer whose CURRENT connection advertised this bit — an old receiver
+ * would fold the raw sentinel VALUE harmlessly (a max value never wins a
+ * min fold) but would latch a spurious same-epoch REGRESSION stall when
+ * the sender wakes and reports a real value again, so old peers keep
+ * receiving the conservative clock sample instead. */
+#define PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 ((uint32)0x00000100U)
 /*
  * PGRAC: spec-7.2 D2 — plane + connection-epoch ride the documented-zero
  * pad region (capabilities precedent: occupy pad bytes, do not resize V1).

src/include/cluster/cluster_sf_dep.h

@@ -201,6 +201,10 @@ extern bool cluster_peer_supports_undo_authority_serve(int32 peer_id);
 /* spec-5.22e D5-2: undo-horizon report capability (connection-bound; see
  * cluster_sf_note_peer_disconnected_gen) + the close-funnel reset hooks. */
 extern bool cluster_sf_peer_supports_undo_horizon(int32 peer_id);
+/* TT lane (S3 idle-peer floor pin): idle-unconstrained sentinel report
+ * capability (connection-bound, same discipline).  SENDER-side gate only:
+ * an old peer keeps receiving the conservative clock sample. */
+extern bool cluster_sf_peer_supports_undo_horizon_idle(int32 peer_id);
 /* GCS-race round-2 review F6: GCS completion-proof capability
  * (connection-bound, same discipline).  An unknown/old/reconnecting peer
  * reads false, which every consumer treats in the SAFE direction: the

src/include/cluster/cluster_undo_horizon.h

@@ -55,6 +55,25 @@
 #define CLUSTER_UNDO_HORIZON_INTERVAL_MIN_MS 100
 #define CLUSTER_UNDO_HORIZON_INTERVAL_MAX_MS 60000
 
+/*
+ * Idle-unconstrained sentinel report value (TT lane, S3 idle-peer floor
+ * pin).  A PROVABLY idle sender (cluster_undo_retention_all_quiescent():
+ * zero active xacts AND zero held snapshots) publishes this instead of its
+ * clock sample, telling the fold "I hold no retention constraint": the
+ * time-axis-min fold ignores it structurally (scn_local == SCN_MAX_LOCAL
+ * never wins a min), so a lone writer's floor becomes its own local
+ * horizon instead of trailing the idle peers' report cadence.  All other
+ * proof obligations (capability, presence, stability, epoch, freshness)
+ * still apply to a sentinel report — an idle peer that stops reporting
+ * still stalls the fold.  A sender that wakes mid-report-window self-fences
+ * through the BELOW_HORIZON read_scn admissibility gate (fail-closed,
+ * observe heals the next snapshot), so rule 8.A polarity is unchanged.
+ * The value deliberately has all node bits set (node id 255 is invalid)
+ * and local == SCN_MAX_LOCAL (the wraparound-guard ceiling): no lawful
+ * clock sample can ever collide with it.
+ */
+#define CLUSTER_UNDO_HORIZON_REPORT_UNCONSTRAINED ((SCN)UINT64_MAX)
+
 typedef enum ClusterUndoHorizonFoldStatus {
 	CLUSTER_UNDO_HORIZON_FOLD_OK = 0,
 	CLUSTER_UNDO_HORIZON_FOLD_STALLED
@@ -208,6 +227,9 @@ extern void cluster_undo_horizon_note_wire_reject(void);
 extern uint64 cluster_undo_horizon_wire_reject_count(void);
 extern void cluster_undo_horizon_note_admission_refuse(void);
 extern uint64 cluster_undo_horizon_admission_refuse_count(void);
+/* TT lane: idle-unconstrained sentinel reports sent (sender-side gauge of
+ * the provably-idle predicate actually firing; one bump per peer send). */
+extern uint64 cluster_undo_horizon_idle_sentinel_sent_count(void);
 extern const char *cluster_undo_horizon_peer_reports_summary(void);
 extern void cluster_undo_horizon_note_floor(SCN scn);
 extern SCN cluster_undo_horizon_last_floor(void);