Fix recovery-conflict wait loop for buffer pins on hot standby

When the startup process waits on a buffer pin during recovery, the
buffer-pin wait code treats any latch wakeup as a relevant event,
resetting all timeouts and retrying. On hot standby, periodic SIGALRM
signals from unrelated timers (such as progress reporting) can arrive
before the deadlock timeout expires, causing the wait loop to reset
indefinitely without ever triggering the deadlock detector. This results
in an infinite loop in LockBufferForCleanup and stalls recovery
indefinitely, particularly when max_standby_streaming_delay = -1.

Author: Vitaly Davydov

src/backend/storage/buffer/bufmgr.c

@@ -3469,6 +3469,25 @@ WakePinCountWaiter(BufferDesc *buf)
 		UnlockBufHdr(buf);
 }
 
+/*
+ * Register the current process as the pincount waiter for a shared buffer,
+ * set the BM_PIN_COUNT_WAITER flag, and unlock the buffer header.
+ *
+ * The caller must hold the buffer header lock, pass the current buffer state
+ * returned by LockBufHdr(), and ensure that no other backend is already
+ * registered as the waiter.
+ */
+static void
+RegisterPinCountWaiter(BufferDesc *bufHdr, uint64 buf_state)
+{
+	Assert((buf_state & BM_PIN_COUNT_WAITER) == 0 ||
+		   bufHdr->wait_backend_pgprocno == MyProcNumber);
+
+	bufHdr->wait_backend_pgprocno = MyProcNumber;
+	PinCountWaitBuf = bufHdr;
+	UnlockBufHdrExt(bufHdr, buf_state, BM_PIN_COUNT_WAITER, 0, 0);
+}
+
 /*
  * UnpinBuffer -- make buffer available for replacement.
  *
@@ -4763,6 +4782,60 @@ BufferGetLSNAtomic(Buffer buffer)
 #endif
 }
 
+/*
+ * PinCountWaiterCheckReadyForCleanup
+ *		Recheck whether pin count waiter process (the startup process)
+ *		can retry cleanup lock acquisition.
+ *
+ * This is only for the hot-standby path in LockBufferForCleanup(), via
+ * ResolveRecoveryConflictWithBufferPin(), after ProcWaitForSignal() returns.
+ * The caller must already be registered as the shared buffer's
+ * BM_PIN_COUNT_WAITER.
+ *
+ * Returns true when the caller itself is the only remaining pin holder, so it
+ * can retry taking the cleanup lock. Returns false if other backends still
+ * pin the shared buffer. In that case, this function guarantees that the
+ * current backend remains registered as the pincount waiter to be woken when
+ * the buffer refcount drops to 1.
+ */
+bool
+PinCountWaiterCheckReadyForCleanup(Buffer buffer)
+{
+	BufferDesc *bufHdr;
+	uint64		buf_state;
+	uint32		buf_refcount;
+
+	Assert(BufferIsValid(buffer));
+	Assert(!BufferIsLocal(buffer));
+
+	bufHdr = GetBufferDescriptor(buffer - 1);
+	Assert(PinCountWaitBuf == bufHdr);
+
+	buf_state = LockBufHdr(bufHdr);
+	buf_refcount = BUF_STATE_GET_REFCOUNT(buf_state);
+
+	if (buf_refcount == 1)
+	{
+		UnlockBufHdr(bufHdr);
+		return true;
+	}
+
+	if ((buf_state & BM_PIN_COUNT_WAITER) != 0 &&
+		bufHdr->wait_backend_pgprocno != MyProcNumber)
+	{
+		UnlockBufHdr(bufHdr);
+		elog(ERROR, "multiple processes attempting to wait for pincount 1");
+	}
+
+	/*
+	 * If other processes still pin the buffer, register this process again as
+	 * the pincount waiter to wait again.
+	 */
+	RegisterPinCountWaiter(bufHdr, buf_state);
+
+	return false;
+}
+
 /* ---------------------------------------------------------------------
  *		DropRelationBuffers
  *

src/backend/storage/ipc/standby.c

@@ -790,22 +790,36 @@ ResolveRecoveryConflictWithLock(LOCKTAG locktag, bool logging_conflict)
  * Deadlocks are extremely rare, and relatively expensive to check for,
  * so we don't do a deadlock check right away ... only if we have had to wait
  * at least deadlock_timeout.
+ *
+ * The current process should be the waiter process and should have
+ * published the waited buffer via SetStartupBufferPinWaitBufId().
  */
 void
 ResolveRecoveryConflictWithBufferPin(void)
 {
 	TimestampTz ltime;
+	int			bufid;
 
 	Assert(InHotStandby);
 
+	bufid = GetStartupBufferPinWaitBufId();
+	Assert(bufid >= 0);
+
 	ltime = GetStandbyLimitTime();
 
-	if (GetCurrentTimestamp() >= ltime && ltime != 0)
+	if (ltime != 0 && GetCurrentTimestamp() >= ltime)
 	{
 		/*
 		 * We're already behind, so clear a path as quickly as possible.
 		 */
 		SendRecoveryConflictWithBufferPin(RECOVERY_CONFLICT_BUFFERPIN);
+
+		/*
+		 * Set delay timeout flag once the timeout is reached (the current
+		 * timestamp is greater than the standby limit time). This variable is
+		 * use by timeout handlers with the same purpose.
+		 */
+		got_standby_delay_timeout = true;
 	}
 	else
 	{
@@ -833,35 +847,53 @@ ResolveRecoveryConflictWithBufferPin(void)
 		enable_timeouts(timeouts, cnt);
 	}
 
-	/*
-	 * Wait to be signaled by UnpinBuffer() or for the wait to be interrupted
-	 * by one of the timeouts established above.
-	 *
-	 * We assume that only UnpinBuffer() and the timeout requests established
-	 * above can wake us up here. WakeupRecovery() called by walreceiver or
-	 * SIGHUP signal handler, etc cannot do that because it uses the different
-	 * latch from that ProcWaitForSignal() waits on.
-	 */
-	ProcWaitForSignal(WAIT_EVENT_BUFFER_CLEANUP);
-
-	if (got_standby_delay_timeout)
-		SendRecoveryConflictWithBufferPin(RECOVERY_CONFLICT_BUFFERPIN);
-	else if (got_standby_deadlock_timeout)
+	for (;;)
 	{
 		/*
-		 * Send out a request for hot-standby backends to check themselves for
-		 * deadlocks.
+		 * Wait to be signaled by UnpinBuffer() or for the wait to be
+		 * interrupted by one of the timeouts established above.
 		 *
-		 * XXX The subsequent ResolveRecoveryConflictWithBufferPin() will wait
-		 * to be signaled by UnpinBuffer() again and send a request for
-		 * deadlocks check if deadlock_timeout happens. This causes the
-		 * request to continue to be sent every deadlock_timeout until the
-		 * buffer is unpinned or ltime is reached. This would increase the
-		 * workload in the startup process and backends. In practice it may
-		 * not be so harmful because the period that the buffer is kept pinned
-		 * is basically no so long. But we should fix this?
+		 * ProcWaitForSignal() can also wake up for unrelated reasons, so
+		 * recheck later whether cleanup can proceed.
 		 */
-		SendRecoveryConflictWithBufferPin(RECOVERY_CONFLICT_BUFFERPIN_DEADLOCK);
+		ProcWaitForSignal(WAIT_EVENT_BUFFER_CLEANUP);
+
+		/*
+		 * Once the reference count is 1, the waiter process itself is the
+		 * only backend pinning the buffer at the moment. There is a chance to
+		 * lock the buffer exclusively.
+		 */
+		if (PinCountWaiterCheckReadyForCleanup(bufid + 1))
+			break;
+
+		/*
+		 * Send the recovery conflict if the standby delay timeout is activated or
+		 * standby limit time was already reached. The second condition handles the
+		 * fast path when timeouts are not activated.
+		 */
+		if (got_standby_delay_timeout)
+		{
+			SendRecoveryConflictWithBufferPin(RECOVERY_CONFLICT_BUFFERPIN);
+			break;
+		}
+		else if (got_standby_deadlock_timeout)
+		{
+			/*
+			 * Send out a request for hot-standby backends to check themselves
+			 * for deadlocks.
+			 *
+			 * XXX The subsequent ResolveRecoveryConflictWithBufferPin() will
+			 * wait to be signaled by UnpinBuffer() again and send a request
+			 * for deadlocks check if deadlock_timeout happens. This causes
+			 * the request to continue to be sent every deadlock_timeout until
+			 * the buffer is unpinned or ltime is reached. This would increase
+			 * the workload in the startup process and backends. In practice
+			 * it may not be so harmful because the period that the buffer is
+			 * kept pinned is basically no so long. But we should fix this?
+			 */
+			SendRecoveryConflictWithBufferPin(RECOVERY_CONFLICT_BUFFERPIN_DEADLOCK);
+			break;
+		}
 	}
 
 	/*

src/include/storage/bufmgr.h

@@ -313,6 +313,7 @@ extern bool BufferIsPermanent(Buffer buffer);
 extern XLogRecPtr BufferGetLSNAtomic(Buffer buffer);
 extern void BufferGetTag(Buffer buffer, RelFileLocator *rlocator,
 						 ForkNumber *forknum, BlockNumber *blknum);
+extern bool PinCountWaiterCheckReadyForCleanup(Buffer buffer);
 
 extern void MarkBufferDirtyHint(Buffer buffer, bool buffer_std);