From a41017e3d44ffe0aca3562a6ff729b2ea8ea9220 Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Mon, 12 Jan 2026 11:49:41 -0500
Subject: [PATCH v13 8/8] Eagerly flush buffer successors

When flushing a dirty buffer, check if the two blocks following it are
in shared buffers and whether or not they are dirty. If they are, flush
them together with the victim buffer.

Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Discussion: https://postgr.es/m/flat/0198DBB9-4A76-49E4-87F8-43D46DD0FD76%40gmail.com#1d8677fc75dc8b39f0eb5dd6bbafb65d
---
 src/backend/storage/buffer/bufmgr.c | 412 ++++++++++++++++------------
 1 file changed, 238 insertions(+), 174 deletions(-)

diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index dd75deb13c1..c185928cb39 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -637,16 +637,16 @@ static BlockNumber WriteBatchInit(BufferDesc *batch_start, uint32 max_batch_size
 static BufferDesc *PrepareOrRejectEagerFlushBuffer(BufferAccessStrategy strategy,
 												   Buffer bufnum,
 												   BufferTag *require,
+												   LWLock *buftable_lock,
 												   XLogRecPtr *lsn);
+static void FindFlushAdjacents(BufferAccessStrategy strategy, BufferDesc *batch_start,
+							   uint32 batch_limit,
+							   BufferWriteBatch *batch);
 static void FindStrategyFlushAdjacents(BufferAccessStrategy strategy, Buffer sweep_end,
 									   BufferDesc *batch_start,
 									   uint32 max_batch_size,
 									   BufferWriteBatch *batch,
 									   int *sweep_cursor);
-static bool PrepareFlushBuffer(BufferDesc *bufdesc, XLogRecPtr *lsn);
-static void DoFlushBuffer(BufferDesc *buf, SMgrRelation reln,
-						  IOObject io_object, IOContext io_context,
-						  XLogRecPtr buffer_lsn);
 static void FindAndDropRelationBuffers(RelFileLocator rlocator,
 									   ForkNumber forkNum,
 									   BlockNumber nForkBlock,
@@ -2494,7 +2494,6 @@ again:
 	if (buf_state & BM_DIRTY)
 	{
 		LWLock	   *content_lock;
-		XLogRecPtr	victim_lsn;
 
 		Assert(buf_state & BM_TAG_VALID);
 		Assert(buf_state & BM_VALID);
@@ -2538,7 +2537,7 @@ again:
 			goto again;
 		}
 
-		if (!PrepareFlushBuffer(buf_hdr, &victim_lsn))
+		if (!StartBufferIO(buf_hdr, false, false))
 		{
 			/* May be nothing to do if buffer was cleaned */
 			LWLockRelease(BufferDescriptorGetContentLock(buf_hdr));
@@ -2571,6 +2570,9 @@ again:
 					BufferWriteBatch batch;
 					BlockNumber limit;
 
+					/* So we can detect block content changes while flushing */
+					pg_atomic_fetch_and_u32(&next_bufdesc->state, ~BM_JUST_DIRTIED);
+
 					/*
 					 * After finding an eligible buffer, if we are allowed
 					 * more pins and there are more blocks in the relation,
@@ -2615,15 +2617,34 @@ again:
 				next_bufdesc = PrepareOrRejectEagerFlushBuffer(strategy,
 															   next_buf,
 															   NULL,
+															   NULL,
 															   &next_buf_lsn);
 			}
 		}
 		else
 		{
-			DoFlushBuffer(buf_hdr, NULL, IOOBJECT_RELATION, io_context, victim_lsn);
-			LWLockRelease(BufferDescriptorGetContentLock(buf_hdr));
-			ScheduleBufferTagForWriteback(&BackendWritebackContext, io_context,
-										  &buf_hdr->tag);
+			BufferWriteBatch batch;
+			BlockNumber limit;
+			uint32		max_batch_size = 3; /* we only look for two successors */
+
+			/* Pin victim again so it stays ours even after batch released */
+			ReservePrivateRefCountEntry();
+			ResourceOwnerEnlarge(CurrentResourceOwner);
+			IncrBufferRefCount(BufferDescriptorGetBuffer(buf_hdr));
+
+			/* So we can detect block content changes while flushing */
+			pg_atomic_fetch_and_u32(&buf_hdr->state, ~BM_JUST_DIRTIED);
+
+			/*
+			 * If we are allowed more pins and there are more blocks in the
+			 * relation and the victim buffer's block's successors are
+			 * eligible for eager flushing, combine them into a batch.
+			 */
+			limit = WriteBatchInit(buf_hdr, max_batch_size, &batch);
+			if (limit > 1)
+				FindFlushAdjacents(strategy, buf_hdr, limit, &batch);
+			FlushBufferBatch(&batch, io_context);
+			CompleteWriteBatchIO(&batch, io_context, &BackendWritebackContext);
 		}
 	}
 
@@ -3547,8 +3568,8 @@ BufferNeedsWALFlush(BufferDesc *bufdesc, bool exclusive_locked)
 	XLogRecPtr	lsn;
 
 	/*
-	 * Unlogged buffers can't need WAL flush. See PrepareFlushBuffer() for
-	 * more details on unlogged relations with LSNs.
+	 * Unlogged buffers can't need WAL flush. See FlushBuffer() for more
+	 * details on unlogged relations with LSNs.
 	 */
 	if (!(buf_state & BM_PERMANENT))
 		return false;
@@ -4724,10 +4745,133 @@ static void
 FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object,
 			IOContext io_context)
 {
-	XLogRecPtr	lsn;
+	XLogRecPtr	recptr;
+	ErrorContextCallback errcallback;
+	instr_time	io_start;
+	Block		bufBlock;
+	char	   *bufToWrite;
+	uint32		buf_state;
+
+	/*
+	 * Try to start an I/O operation.  If StartBufferIO returns false, then
+	 * someone else flushed the buffer before we could, so we need not do
+	 * anything.
+	 */
+	if (!StartBufferIO(buf, false, false))
+		return;
+
+	/* Setup error traceback support for ereport() */
+	errcallback.callback = shared_buffer_write_error_callback;
+	errcallback.arg = buf;
+	errcallback.previous = error_context_stack;
+	error_context_stack = &errcallback;
+
+	/* Find smgr relation for buffer */
+	if (reln == NULL)
+		reln = smgropen(BufTagGetRelFileLocator(&buf->tag), INVALID_PROC_NUMBER);
+
+	TRACE_POSTGRESQL_BUFFER_FLUSH_START(BufTagGetForkNum(&buf->tag),
+										buf->tag.blockNum,
+										reln->smgr_rlocator.locator.spcOid,
+										reln->smgr_rlocator.locator.dbOid,
+										reln->smgr_rlocator.locator.relNumber);
+
+	buf_state = LockBufHdr(buf);
+
+	/*
+	 * Run PageGetLSN while holding header lock, since we don't have the
+	 * buffer locked exclusively in all cases.
+	 */
+	recptr = BufferGetLSN(buf);
+
+	/* To check if block content changes while flushing. - vadim 01/17/97 */
+	UnlockBufHdrExt(buf, buf_state,
+					0, BM_JUST_DIRTIED,
+					0);
+
+	/*
+	 * Force XLOG flush up to buffer's LSN.  This implements the basic WAL
+	 * rule that log updates must hit disk before any of the data-file changes
+	 * they describe do.
+	 *
+	 * However, this rule does not apply to unlogged relations, which will be
+	 * lost after a crash anyway.  Most unlogged relation pages do not bear
+	 * LSNs since we never emit WAL records for them, and therefore flushing
+	 * up through the buffer LSN would be useless, but harmless.  However,
+	 * GiST indexes use LSNs internally to track page-splits, and therefore
+	 * unlogged GiST pages bear "fake" LSNs generated by
+	 * GetFakeLSNForUnloggedRel.  It is unlikely but possible that the fake
+	 * LSN counter could advance past the WAL insertion point; and if it did
+	 * happen, attempting to flush WAL through that location would fail, with
+	 * disastrous system-wide consequences.  To make sure that can't happen,
+	 * skip the flush if the buffer isn't permanent.
+	 */
+	if (buf_state & BM_PERMANENT)
+		XLogFlush(recptr);
+
+	/*
+	 * Now it's safe to write the buffer to disk. Note that no one else should
+	 * have been able to write it, while we were busy with log flushing,
+	 * because we got the exclusive right to perform I/O by setting the
+	 * BM_IO_IN_PROGRESS bit.
+	 */
+	bufBlock = BufHdrGetBlock(buf);
+
+	/*
+	 * Update page checksum if desired.  Since we have only shared lock on the
+	 * buffer, other processes might be updating hint bits in it, so we must
+	 * copy the page to private storage if we do checksumming.
+	 */
+	bufToWrite = PageSetChecksumCopy((Page) bufBlock, buf->tag.blockNum);
+
+	io_start = pgstat_prepare_io_time(track_io_timing);
+
+	/*
+	 * bufToWrite is either the shared buffer or a copy, as appropriate.
+	 */
+	smgrwrite(reln,
+			  BufTagGetForkNum(&buf->tag),
+			  buf->tag.blockNum,
+			  bufToWrite,
+			  false);
+
+	/*
+	 * When a strategy is in use, only flushes of dirty buffers already in the
+	 * strategy ring are counted as strategy writes (IOCONTEXT
+	 * [BULKREAD|BULKWRITE|VACUUM] IOOP_WRITE) for the purpose of IO
+	 * statistics tracking.
+	 *
+	 * If a shared buffer initially added to the ring must be flushed before
+	 * being used, this is counted as an IOCONTEXT_NORMAL IOOP_WRITE.
+	 *
+	 * If a shared buffer which was added to the ring later because the
+	 * current strategy buffer is pinned or in use or because all strategy
+	 * buffers were dirty and rejected (for BAS_BULKREAD operations only)
+	 * requires flushing, this is counted as an IOCONTEXT_NORMAL IOOP_WRITE
+	 * (from_ring will be false).
+	 *
+	 * When a strategy is not in use, the write can only be a "regular" write
+	 * of a dirty shared buffer (IOCONTEXT_NORMAL IOOP_WRITE).
+	 */
+	pgstat_count_io_op_time(IOOBJECT_RELATION, io_context,
+							IOOP_WRITE, io_start, 1, BLCKSZ);
+
+	pgBufferUsage.shared_blks_written++;
+
+	/*
+	 * Mark the buffer as clean (unless BM_JUST_DIRTIED has become set) and
+	 * end the BM_IO_IN_PROGRESS state.
+	 */
+	TerminateBufferIO(buf, true, 0, true, false);
+
+	TRACE_POSTGRESQL_BUFFER_FLUSH_DONE(BufTagGetForkNum(&buf->tag),
+									   buf->tag.blockNum,
+									   reln->smgr_rlocator.locator.spcOid,
+									   reln->smgr_rlocator.locator.dbOid,
+									   reln->smgr_rlocator.locator.relNumber);
 
-	if (PrepareFlushBuffer(buf, &lsn))
-		DoFlushBuffer(buf, reln, io_object, io_context, lsn);
+	/* Pop the error context stack */
+	error_context_stack = errcallback.previous;
 }
 
 /*
@@ -4776,12 +4920,17 @@ WriteBatchInit(BufferDesc *batch_start, uint32 max_batch_size,
  * accept it, they will provide the required block number and its
  * RelFileLocator and fork.
  *
- * If returning a buffer, also return its LSN.
+ * If the caller is holding the buftable_lock, it will be released after
+ * acquiring a pin on the buffer.
+ *
+ * max_lsn may be updated if the provided buffer LSN exceeds the current max
+ * LSN.
  */
 static BufferDesc *
 PrepareOrRejectEagerFlushBuffer(BufferAccessStrategy strategy,
 								Buffer bufnum,
 								BufferTag *require,
+								LWLock *buftable_lock,
 								XLogRecPtr *lsn)
 {
 	BufferDesc *bufdesc;
@@ -4843,6 +4992,12 @@ PrepareOrRejectEagerFlushBuffer(BufferAccessStrategy strategy,
 	if (!PinBuffer(bufdesc, strategy, /* skip_if_not_valid */ true))
 		goto reject_buffer;
 
+	if (buftable_lock)
+	{
+		LWLockRelease(buftable_lock);
+		buftable_lock = NULL;
+	}
+
 	CheckBufferIsPinnedOnce(bufnum);
 
 	/* Now that we have the buffer pinned, recheck it's got the right block */
@@ -4875,6 +5030,8 @@ reject_buffer_unpin:
 	UnpinBuffer(bufdesc);
 
 reject_buffer:
+	if (buftable_lock)
+		LWLockRelease(buftable_lock);
 	return NULL;
 }
 
@@ -4883,15 +5040,17 @@ reject_buffer:
  * flushing, find additional buffers from the ring that can be combined into a
  * single write batch with the starting buffer.
  *
- * This function will pin and content lock all of the buffers that it
- * assembles for the IO batch. The caller is responsible for issuing the IO.
- *
- * batch_limit is the largest batch we are allowed to construct given the
- * remaining blocks in the table, the number of available pins, and the
- * current configuration.
+ * max_batch_size is the maximum number of blocks that can be combined into a
+ * single write in general. This function, based on the block number of start,
+ * will determine the maximum IO size for this particular write given how much
+ * of the file remains. max_batch_size is provided by the caller so it doesn't
+ * have to be recalculated for each write.
  *
  * batch is an output parameter that this function will fill with the needed
  * information to issue this IO.
+ *
+ * This function will pin and content lock all of the buffers that it
+ * assembles for the IO batch. The caller is responsible for issuing the IO.
  */
 static void
 FindStrategyFlushAdjacents(BufferAccessStrategy strategy,
@@ -4930,6 +5089,7 @@ FindStrategyFlushAdjacents(BufferAccessStrategy strategy,
 		batch->bufdescs[batch->n] =
 			PrepareOrRejectEagerFlushBuffer(strategy, bufnum,
 											&require,
+											NULL,
 											&lsn);
 
 		/*
@@ -4945,7 +5105,63 @@ FindStrategyFlushAdjacents(BufferAccessStrategy strategy,
 		/* Stop when we encounter a buffer that will break the run */
 		if (batch->bufdescs[batch->n] == NULL)
 			break;
+	}
+}
+
+
+/*
+ * Check if the blocks after my block are in shared buffers and dirty and if
+ * they are, write them out too.
+ */
+static void
+FindFlushAdjacents(BufferAccessStrategy strategy, BufferDesc *batch_start,
+				   uint32 batch_limit,
+				   BufferWriteBatch *batch)
+{
+	BufferTag	newTag;			/* identity of requested block */
+	uint32		newHash;		/* hash value for newTag */
+	LWLock	   *newPartitionLock;	/* buffer partition lock for it */
+	int			buf_id;
+	BufferTag	require;
+
+	/* create a tag so we can lookup the buffers */
+	InitBufferTag(&require, &batch->reln->smgr_rlocator.locator,
+				  batch->forkno, InvalidBlockNumber);
+
+	for (; batch->n < batch_limit; batch->n++)
+	{
+		XLogRecPtr	lsn;
+
+		require.blockNum = batch->start + batch->n;
+
+		Assert(BlockNumberIsValid(require.blockNum));
+
+		/* determine its hash code and partition lock ID */
+		newHash = BufTableHashCode(&newTag);
+		newPartitionLock = BufMappingPartitionLock(newHash);
+
+		/* see if the block is in the buffer pool already */
+		LWLockAcquire(newPartitionLock, LW_SHARED);
+		buf_id = BufTableLookup(&newTag, newHash);
+
+		/* The block may not even be in shared buffers. */
+		if (buf_id < 0)
+		{
+			LWLockRelease(newPartitionLock);
+			break;
+		}
 
+		batch->bufdescs[batch->n] =
+			PrepareOrRejectEagerFlushBuffer(strategy,
+											buf_id + 1,
+											&require,
+											newPartitionLock,
+											&lsn);
+		if (lsn > batch->max_lsn)
+			batch->max_lsn = lsn;
+
+		if (batch->bufdescs[batch->n] == NULL)
+			break;
 	}
 }
 
@@ -5009,158 +5225,6 @@ FlushBufferBatch(BufferWriteBatch *batch,
 	error_context_stack = errcallback.previous;
 }
 
-/*
- * Prepare the buffer with bufdesc for writing. Returns true if the buffer
- * actually needs writing and false otherwise. lsn returns the buffer's LSN if
- * the table is logged and still needs flushing.
- */
-static bool
-PrepareFlushBuffer(BufferDesc *bufdesc, XLogRecPtr *lsn)
-{
-	uint32		buf_state;
-
-	*lsn = InvalidXLogRecPtr;
-
-	/*
-	 * Try to start an I/O operation.  If StartBufferIO returns false, then
-	 * someone else flushed the buffer before we could, so we need not do
-	 * anything.
-	 */
-	if (!StartBufferIO(bufdesc, false, false))
-		return false;
-
-	buf_state = LockBufHdr(bufdesc);
-
-	/*
-	 * Record the buffer's LSN. We will force XLOG flush up to buffer's LSN.
-	 * This implements the basic WAL rule that log updates must hit disk
-	 * before any of the data-file changes they describe do.
-	 *
-	 * However, this rule does not apply to unlogged relations, which will be
-	 * lost after a crash anyway.  Most unlogged relation pages do not bear
-	 * LSNs since we never emit WAL records for them, and therefore flushing
-	 * up through the buffer LSN would be useless, but harmless.  However,
-	 * GiST indexes use LSNs internally to track page-splits, and therefore
-	 * unlogged GiST pages bear "fake" LSNs generated by
-	 * GetFakeLSNForUnloggedRel.  It is unlikely but possible that the fake
-	 * LSN counter could advance past the WAL insertion point; and if it did
-	 * happen, attempting to flush WAL through that location would fail, with
-	 * disastrous system-wide consequences.  To make sure that can't happen,
-	 * skip the flush if the buffer isn't permanent.
-	 *
-	 * We must hold the buffer header lock when examining the page LSN since
-	 * we don't have buffer exclusively locked in all cases.
-	 */
-	if (buf_state & BM_PERMANENT)
-		*lsn = BufferGetLSN(bufdesc);
-
-	/* To check if block content changes while flushing. - vadim 01/17/97 */
-	UnlockBufHdrExt(bufdesc, buf_state,
-					0, BM_JUST_DIRTIED,
-					0);
-	return true;
-}
-
-/*
- * Actually do the write I/O to clean a buffer. buf and reln may be modified.
- */
-static void
-DoFlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object,
-			  IOContext io_context, XLogRecPtr buffer_lsn)
-{
-	ErrorContextCallback errcallback;
-	instr_time	io_start;
-	Block		bufBlock;
-	char	   *bufToWrite;
-
-	/* Setup error traceback support for ereport() */
-	errcallback.callback = shared_buffer_write_error_callback;
-	errcallback.arg = buf;
-	errcallback.previous = error_context_stack;
-	error_context_stack = &errcallback;
-
-	/* Find smgr relation for buffer */
-	if (reln == NULL)
-		reln = smgropen(BufTagGetRelFileLocator(&buf->tag), INVALID_PROC_NUMBER);
-
-	TRACE_POSTGRESQL_BUFFER_FLUSH_START(BufTagGetForkNum(&buf->tag),
-										buf->tag.blockNum,
-										reln->smgr_rlocator.locator.spcOid,
-										reln->smgr_rlocator.locator.dbOid,
-										reln->smgr_rlocator.locator.relNumber);
-
-	/* Force XLOG flush up to buffer's LSN */
-	if (XLogRecPtrIsValid(buffer_lsn))
-	{
-		Assert(pg_atomic_read_u32(&buf->state) & BM_PERMANENT);
-		XLogFlush(buffer_lsn);
-	}
-
-	/*
-	 * Now it's safe to write the buffer to disk. Note that no one else should
-	 * have been able to write it, while we were busy with log flushing,
-	 * because we got the exclusive right to perform I/O by setting the
-	 * BM_IO_IN_PROGRESS bit.
-	 */
-	bufBlock = BufHdrGetBlock(buf);
-
-	/*
-	 * Update page checksum if desired.  Since we have only shared lock on the
-	 * buffer, other processes might be updating hint bits in it, so we must
-	 * copy the page to private storage if we do checksumming.
-	 */
-	bufToWrite = PageSetChecksumCopy((Page) bufBlock, buf->tag.blockNum);
-
-	io_start = pgstat_prepare_io_time(track_io_timing);
-
-	/*
-	 * bufToWrite is either the shared buffer or a copy, as appropriate.
-	 */
-	smgrwrite(reln,
-			  BufTagGetForkNum(&buf->tag),
-			  buf->tag.blockNum,
-			  bufToWrite,
-			  false);
-
-	/*
-	 * When a strategy is in use, only flushes of dirty buffers already in the
-	 * strategy ring are counted as strategy writes (IOCONTEXT
-	 * [BULKREAD|BULKWRITE|VACUUM] IOOP_WRITE) for the purpose of IO
-	 * statistics tracking.
-	 *
-	 * If a shared buffer initially added to the ring must be flushed before
-	 * being used, this is counted as an IOCONTEXT_NORMAL IOOP_WRITE.
-	 *
-	 * If a shared buffer which was added to the ring later because the
-	 * current strategy buffer is pinned or in use or because all strategy
-	 * buffers were dirty and rejected (for BAS_BULKREAD operations only)
-	 * requires flushing, this is counted as an IOCONTEXT_NORMAL IOOP_WRITE
-	 * (from_ring will be false).
-	 *
-	 * When a strategy is not in use, the write can only be a "regular" write
-	 * of a dirty shared buffer (IOCONTEXT_NORMAL IOOP_WRITE).
-	 */
-	pgstat_count_io_op_time(IOOBJECT_RELATION, io_context,
-							IOOP_WRITE, io_start, 1, BLCKSZ);
-
-	pgBufferUsage.shared_blks_written++;
-
-	/*
-	 * Mark the buffer as clean (unless BM_JUST_DIRTIED has become set) and
-	 * end the BM_IO_IN_PROGRESS state.
-	 */
-	TerminateBufferIO(buf, true, 0, true, false);
-
-	TRACE_POSTGRESQL_BUFFER_FLUSH_DONE(BufTagGetForkNum(&buf->tag),
-									   buf->tag.blockNum,
-									   reln->smgr_rlocator.locator.spcOid,
-									   reln->smgr_rlocator.locator.dbOid,
-									   reln->smgr_rlocator.locator.relNumber);
-
-	/* Pop the error context stack */
-	error_context_stack = errcallback.previous;
-}
-
 /*
  * Convenience wrapper around FlushBuffer() that locks/unlocks the buffer
  * before/after calling FlushBuffer().
-- 
2.43.0

