From 622f55505ed2eac23cfa3ce4ece0dc5936fc880b Mon Sep 17 00:00:00 2001
From: Melanie Plageman <melanieplageman@gmail.com>
Date: Wed, 15 Oct 2025 13:42:47 -0400
Subject: [PATCH v13 4/8] Write combining for BAS_BULKWRITE

Implement write combining for users of the bulkwrite buffer access
strategy (e.g. COPY FROM). When the buffer access strategy needs to
clean a buffer for reuse, it already opportunistically flushes some
other buffers. Now, combine any contiguous blocks from the same relation
into larger writes and issue them with smgrwritev().

The performance benefit for COPY FROM is mostly noticeable for multiple
concurrent COPY FROMs because a single COPY FROM is either CPU bound or
bound by WAL writes.

The infrastructure for flushing larger batches of IOs will be reused by
checkpointer and other processes doing writes of dirty data.

XXX: Because this sets in-place checksums for batches, it is not
committable until additional infrastructure goes in place.

Author: Melanie Plageman <melanieplageman@gmail.com>
Reviewed-by: Chao Li <li.evan.chao@gmail.com>
Discussion: https://postgr.es/m/flat/CAAKRu_bcWRvRwZUop_d9vzF9nHAiT%2B-uPzkJ%3DS3ShZ1GqeAYOw%40mail.gmail.com
---
 src/backend/storage/buffer/bufmgr.c | 315 ++++++++++++++++++++++++++--
 src/backend/storage/page/bufpage.c  |  20 ++
 src/backend/utils/probes.d          |   2 +
 src/include/storage/buf_internals.h |  30 +++
 src/include/storage/bufpage.h       |   2 +
 src/tools/pgindent/typedefs.list    |   1 +
 6 files changed, 354 insertions(+), 16 deletions(-)

diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index cab552bbc56..8a1a1b92739 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -626,15 +626,24 @@ static inline BufferDesc *BufferAlloc(SMgrRelation smgr,
 									  bool *foundPtr, IOContext io_context);
 static bool AsyncReadBuffers(ReadBuffersOperation *operation, int *nblocks_progress);
 static void CheckReadBuffersOperation(ReadBuffersOperation *operation, bool is_complete);
+static uint32 MaxWriteBatchSize(BufferAccessStrategy strategy);
 static Buffer GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context);
 static void FlushUnlockedBuffer(BufferDesc *buf, SMgrRelation reln,
 								IOObject io_object, IOContext io_context);
 static void FlushBuffer(BufferDesc *buf, SMgrRelation reln,
 						IOObject io_object, IOContext io_context);
 
+static BlockNumber WriteBatchInit(BufferDesc *batch_start, uint32 max_batch_size,
+								  BufferWriteBatch *batch);
 static BufferDesc *PrepareOrRejectEagerFlushBuffer(BufferAccessStrategy strategy,
 												   Buffer bufnum,
+												   BufferTag *require,
 												   XLogRecPtr *lsn);
+static void FindStrategyFlushAdjacents(BufferAccessStrategy strategy, Buffer sweep_end,
+									   BufferDesc *batch_start,
+									   uint32 max_batch_size,
+									   BufferWriteBatch *batch,
+									   int *sweep_cursor);
 static bool PrepareFlushBuffer(BufferDesc *bufdesc, XLogRecPtr *lsn);
 static void DoFlushBuffer(BufferDesc *buf, SMgrRelation reln,
 						  IOObject io_object, IOContext io_context,
@@ -2418,6 +2427,34 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr)
 	return true;
 }
 
+/*
+ * Determine the largest IO we can assemble given strategy-specific and global
+ * constraints on the number of pinned buffers and max IO size. Currently only
+ * a single write is inflight at a time, so the batch can consume all the
+ * pinned buffers this backend is allowed. Only for batches of shared
+ * (non-local) relations.
+ */
+static uint32
+MaxWriteBatchSize(BufferAccessStrategy strategy)
+{
+	uint32		result = io_combine_limit;
+	uint32		strategy_pin_limit;
+	uint32		max_pin_limit = GetPinLimit();
+
+	/* Apply pin limits */
+	result = Min(result, max_pin_limit);
+	if (strategy)
+	{
+		strategy_pin_limit = GetAccessStrategyPinLimit(strategy);
+		result = Min(result, strategy_pin_limit);
+	}
+
+	/* Ensure forward progress */
+	result = Max(result, 1);
+
+	return result;
+}
+
 static Buffer
 GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context)
 {
@@ -2511,27 +2548,48 @@ again:
 		{
 			Buffer		sweep_end = buf;
 			int			cursor = StrategyGetCurrentIndex(strategy);
-			bool		first_buffer = true;
+			uint32		max_batch_size = MaxWriteBatchSize(strategy);
 			BufferDesc *next_bufdesc = buf_hdr;
 
+			/* Pin victim again so it stays ours even after batch released */
+			ReservePrivateRefCountEntry();
+			ResourceOwnerEnlarge(CurrentResourceOwner);
+			IncrBufferRefCount(BufferDescriptorGetBuffer(buf_hdr));
+
 			/*
 			 * Flush the victim buffer and then loop around strategy ring one
-			 * time eagerly flushing all of the eligible buffers.
+			 * time eagerly flushing all of the eligible buffers. IO
+			 * concurrency only needs to be taken into account if AIO writes
+			 * are added in the future.
 			 */
 			for (;;)
 			{
 				Buffer		next_buf;
+				XLogRecPtr	next_buf_lsn;	/* unused */
 
 				if (next_bufdesc)
 				{
-					DoFlushBuffer(next_bufdesc, NULL, IOOBJECT_RELATION, io_context, victim_lsn);
-					LWLockRelease(BufferDescriptorGetContentLock(next_bufdesc));
-					ScheduleBufferTagForWriteback(&BackendWritebackContext, io_context,
-												  &next_bufdesc->tag);
-					/* We leave the first buffer pinned for the caller */
-					if (!first_buffer)
-						UnpinBuffer(next_bufdesc);
-					first_buffer = false;
+					BufferWriteBatch batch;
+					BlockNumber limit;
+
+					/*
+					 * After finding an eligible buffer, if we are allowed
+					 * more pins and there are more blocks in the relation,
+					 * identify any of the buffers following it which are also
+					 * eligible and combine them into a batch.
+					 *
+					 * The cursor will be advanced through the ring such that
+					 * the next write batch will start at the next eligible
+					 * buffer after the current batch ends.
+					 */
+					limit = WriteBatchInit(next_bufdesc, max_batch_size, &batch);
+					if (limit > 1)
+						FindStrategyFlushAdjacents(strategy, sweep_end,
+												   next_bufdesc,
+												   limit, &batch, &cursor);
+					FlushBufferBatch(&batch, io_context);
+					/* Pins and locks released inside CompleteWriteBatchIO */
+					CompleteWriteBatchIO(&batch, io_context, &BackendWritebackContext);
 				}
 
 				next_buf = StrategyNextBuffer(strategy, &cursor);
@@ -2551,13 +2609,14 @@ again:
 					break;
 
 				/*
-				 * Check buffer eager flush eligibility. If the buffer is
-				 * ineligible, we'll keep looking until we complete one full
-				 * sweep around the ring.
+				 * If the buffer is eligible for eager flushing, it will be
+				 * the start of a new batch.  Otherwise, we'll keep looking
+				 * until we complete one full sweep around the ring.
 				 */
 				next_bufdesc = PrepareOrRejectEagerFlushBuffer(strategy,
 															   next_buf,
-															   &victim_lsn);
+															   NULL,
+															   &next_buf_lsn);
 			}
 		}
 		else
@@ -4480,6 +4539,41 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object,
 		DoFlushBuffer(buf, reln, io_object, io_context, lsn);
 }
 
+/*
+ * Given the starting buffer of a batch, initialize the batch structure. The
+ * starting buffer must be ready to flush.
+ */
+static BlockNumber
+WriteBatchInit(BufferDesc *batch_start, uint32 max_batch_size,
+			   BufferWriteBatch *batch)
+{
+	BlockNumber limit;
+	uint32		buf_state;
+
+	Assert(batch_start);
+	batch->bufdescs[0] = batch_start;
+
+	Assert(BufferIsLockedByMe(BufferDescriptorGetBuffer(batch_start)));
+	buf_state = LockBufHdr(batch_start);
+	batch->max_lsn = buf_state & BM_PERMANENT ?
+		BufferGetLSN(batch_start) : InvalidXLogRecPtr;
+	UnlockBufHdr(batch_start);
+
+	batch->start = batch->bufdescs[0]->tag.blockNum;
+	Assert(BlockNumberIsValid(batch->start));
+	batch->forkno = BufTagGetForkNum(&batch->bufdescs[0]->tag);
+	batch->reln = smgropen(BufTagGetRelFileLocator(&batch->bufdescs[0]->tag),
+						   INVALID_PROC_NUMBER);
+
+	batch->n = 1;
+
+	limit = smgrmaxcombine(batch->reln, batch->forkno, batch->start);
+	limit = Min(max_batch_size, limit);
+	limit = Min(GetAdditionalPinLimit(), limit);
+
+	return limit;
+}
+
 /*
  * Prepare bufdesc for eager flushing.
  *
@@ -4487,11 +4581,16 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object,
  * pinned and locked and with BM_IO_IN_PROGRESS set, or NULL if this buffer
  * does not contain a block that should be flushed.
  *
+ * If the caller requires a particular block to be in the buffer in order to
+ * accept it, they will provide the required block number and its
+ * RelFileLocator and fork.
+ *
  * If returning a buffer, also return its LSN.
  */
 static BufferDesc *
 PrepareOrRejectEagerFlushBuffer(BufferAccessStrategy strategy,
 								Buffer bufnum,
+								BufferTag *require,
 								XLogRecPtr *lsn)
 {
 	BufferDesc *bufdesc;
@@ -4506,11 +4605,19 @@ PrepareOrRejectEagerFlushBuffer(BufferAccessStrategy strategy,
 	Assert(!BufferIsLocal(bufnum));
 
 	bufdesc = GetBufferDescriptor(bufnum - 1);
+
+	/*
+	 * Quick, unsafe checks to see if buffer even possibly contains a block
+	 * meeting our requirements. We'll recheck it all again after getting a
+	 * pin.
+	 */
+	if (require && !BufferTagsEqual(require, &bufdesc->tag))
+		goto reject_buffer;
+
 	buf_state = pg_atomic_read_u32(&bufdesc->state);
 
 	/*
-	 * Quick racy check to see if the buffer is clean, in which case we don't
-	 * need to flush it. We'll recheck if it is dirty again later before
+	 * We'll recheck if it is dirty later, when we have a pin and lock, before
 	 * actually setting BM_IO_IN_PROGRESS.
 	 */
 	if (!(buf_state & BM_DIRTY))
@@ -4547,6 +4654,10 @@ PrepareOrRejectEagerFlushBuffer(BufferAccessStrategy strategy,
 
 	CheckBufferIsPinnedOnce(bufnum);
 
+	/* Now that we have the buffer pinned, recheck it's got the right block */
+	if (require && !BufferTagsEqual(require, &bufdesc->tag))
+		goto reject_buffer_unpin;
+
 	content_lock = BufferDescriptorGetContentLock(bufdesc);
 	if (!LWLockConditionalAcquire(content_lock, LW_SHARED))
 		goto reject_buffer_unpin;
@@ -4576,6 +4687,137 @@ reject_buffer:
 	return NULL;
 }
 
+/*
+ * Given a starting buffer descriptor from a strategy ring that supports eager
+ * flushing, find additional buffers from the ring that can be combined into a
+ * single write batch with the starting buffer.
+ *
+ * This function will pin and content lock all of the buffers that it
+ * assembles for the IO batch. The caller is responsible for issuing the IO.
+ *
+ * batch_limit is the largest batch we are allowed to construct given the
+ * remaining blocks in the table, the number of available pins, and the
+ * current configuration.
+ *
+ * batch is an output parameter that this function will fill with the needed
+ * information to issue this IO.
+ */
+static void
+FindStrategyFlushAdjacents(BufferAccessStrategy strategy,
+						   Buffer sweep_end,
+						   BufferDesc *batch_start,
+						   uint32 batch_limit,
+						   BufferWriteBatch *batch,
+						   int *sweep_cursor)
+{
+	BufferTag	require;
+
+	Assert(batch_limit > 1);
+
+	InitBufferTag(&require, &batch->reln->smgr_rlocator.locator,
+				  batch->forkno, InvalidBlockNumber);
+
+	/* Now assemble a run of blocks to write out. */
+	for (; batch->n < batch_limit; batch->n++)
+	{
+		Buffer		bufnum;
+		XLogRecPtr	lsn;
+
+		if ((bufnum =
+			 StrategyNextBuffer(strategy, sweep_cursor)) == sweep_end)
+			break;
+
+		/*
+		 * For BAS_BULKWRITE, once you hit an InvalidBuffer, the remaining
+		 * buffers in the ring will be invalid.
+		 */
+		if (!BufferIsValid(bufnum))
+			break;
+
+		require.blockNum = batch->start + batch->n;
+
+		batch->bufdescs[batch->n] =
+			PrepareOrRejectEagerFlushBuffer(strategy, bufnum,
+											&require,
+											&lsn);
+
+		/*
+		 * Because we don't eagerly flush buffers that need WAL flushed, this
+		 * buffer's LSN should only be greater than the victim buffer LSN if
+		 * the victim doesn't need WAL flushing either -- in which case, we
+		 * don't really need to update max_lsn. But, it seems better to keep
+		 * the max_lsn honest -- especially since doing so is cheap.
+		 */
+		if (lsn > batch->max_lsn)
+			batch->max_lsn = lsn;
+
+		/* Stop when we encounter a buffer that will break the run */
+		if (batch->bufdescs[batch->n] == NULL)
+			break;
+
+	}
+}
+
+/*
+ * Given a prepared batch of buffers write them out as a vector.
+ */
+void
+FlushBufferBatch(BufferWriteBatch *batch,
+				 IOContext io_context)
+{
+	BlockNumber blknums[MAX_IO_COMBINE_LIMIT];
+	Block		blocks[MAX_IO_COMBINE_LIMIT];
+	instr_time	io_start;
+	ErrorContextCallback errcallback =
+	{
+		.callback = shared_buffer_write_error_callback,
+		.previous = error_context_stack,
+	};
+
+	error_context_stack = &errcallback;
+
+	if (XLogRecPtrIsValid(batch->max_lsn))
+		XLogFlush(batch->max_lsn);
+
+	/* Should have been opened when initializing the batch */
+	Assert(batch->reln);
+
+#ifdef USE_ASSERT_CHECKING
+	for (uint32 i = 0; i < batch->n; i++)
+		Assert(!BufferNeedsWALFlush(batch->bufdescs[i], false));
+#endif
+
+	TRACE_POSTGRESQL_BUFFER_BATCH_FLUSH_START(batch->forkno,
+											  batch->reln->smgr_rlocator.locator.spcOid,
+											  batch->reln->smgr_rlocator.locator.dbOid,
+											  batch->reln->smgr_rlocator.locator.relNumber,
+											  batch->reln->smgr_rlocator.backend,
+											  batch->n);
+
+	/*
+	 * XXX: All blocks should be copied and then checksummed but doing so
+	 * takes a lot of extra memory and a future patch will eliminate this
+	 * requirement.
+	 */
+	for (BlockNumber i = 0; i < batch->n; i++)
+	{
+		blknums[i] = batch->start + i;
+		blocks[i] = BufHdrGetBlock(batch->bufdescs[i]);
+	}
+
+	PageSetBatchChecksumInplace((Page *) blocks, blknums, batch->n);
+
+	io_start = pgstat_prepare_io_time(track_io_timing);
+
+	smgrwritev(batch->reln, batch->forkno,
+			   batch->start, (const void **) blocks, batch->n, false);
+
+	pgstat_count_io_op_time(IOOBJECT_RELATION, io_context, IOOP_WRITE,
+							io_start, batch->n, BLCKSZ);
+
+	error_context_stack = errcallback.previous;
+}
+
 /*
  * Prepare the buffer with bufdesc for writing. Returns true if the buffer
  * actually needs writing and false otherwise. lsn returns the buffer's LSN if
@@ -4741,6 +4983,47 @@ FlushUnlockedBuffer(BufferDesc *buf, SMgrRelation reln,
 	LWLockRelease(BufferDescriptorGetContentLock(buf));
 }
 
+/*
+ * Given a previously initialized batch with buffers that have already been
+ * flushed, terminate the IO on each buffer and then unlock and unpin them.
+ * This assumes all the buffers were locked and pinned. wb_context will be
+ * modified.
+ */
+void
+CompleteWriteBatchIO(BufferWriteBatch *batch, IOContext io_context,
+					 WritebackContext *wb_context)
+{
+	ErrorContextCallback errcallback =
+	{
+		.callback = shared_buffer_write_error_callback,
+		.previous = error_context_stack,
+	};
+
+	error_context_stack = &errcallback;
+	pgBufferUsage.shared_blks_written += batch->n;
+
+	for (uint32 i = 0; i < batch->n; i++)
+	{
+		Buffer		buffer = BufferDescriptorGetBuffer(batch->bufdescs[i]);
+
+		errcallback.arg = batch->bufdescs[i];
+
+		/* Mark the buffer as clean and end the BM_IO_IN_PROGRESS state. */
+		TerminateBufferIO(batch->bufdescs[i], true, 0, true, false);
+		UnlockReleaseBuffer(buffer);
+		ScheduleBufferTagForWriteback(wb_context, io_context,
+									  &batch->bufdescs[i]->tag);
+	}
+
+	TRACE_POSTGRESQL_BUFFER_BATCH_FLUSH_DONE(batch->forkno,
+											 batch->reln->smgr_rlocator.locator.spcOid,
+											 batch->reln->smgr_rlocator.locator.dbOid,
+											 batch->reln->smgr_rlocator.locator.relNumber,
+											 batch->reln->smgr_rlocator.backend,
+											 batch->n, batch->start);
+	error_context_stack = errcallback.previous;
+}
+
 /*
  * RelationGetNumberOfBlocksInFork
  *		Determines the current number of pages in the specified relation fork.
diff --git a/src/backend/storage/page/bufpage.c b/src/backend/storage/page/bufpage.c
index de85911e3ac..4d0f1883a26 100644
--- a/src/backend/storage/page/bufpage.c
+++ b/src/backend/storage/page/bufpage.c
@@ -1546,3 +1546,23 @@ PageSetChecksumInplace(Page page, BlockNumber blkno)
 
 	((PageHeader) page)->pd_checksum = pg_checksum_page(page, blkno);
 }
+
+/*
+ * A helper to set multiple blocks' checksums
+ */
+void
+PageSetBatchChecksumInplace(Page *pages, const BlockNumber *blknos, uint32 length)
+{
+	/* If we don't need a checksum, just return */
+	if (!DataChecksumsEnabled())
+		return;
+
+	for (uint32 i = 0; i < length; i++)
+	{
+		Page		page = pages[i];
+
+		if (PageIsNew(page))
+			continue;
+		((PageHeader) page)->pd_checksum = pg_checksum_page(page, blknos[i]);
+	}
+}
diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d
index 1929521c6a5..e0f48c6d2d9 100644
--- a/src/backend/utils/probes.d
+++ b/src/backend/utils/probes.d
@@ -61,6 +61,8 @@ provider postgresql {
 	probe buffer__flush__done(ForkNumber, BlockNumber, Oid, Oid, Oid);
 	probe buffer__extend__start(ForkNumber, Oid, Oid, Oid, int, unsigned int);
 	probe buffer__extend__done(ForkNumber, Oid, Oid, Oid, int, unsigned int, BlockNumber);
+	probe buffer__batch__flush__start(ForkNumber, Oid, Oid, Oid, int, unsigned int);
+	probe buffer__batch__flush__done(ForkNumber, Oid, Oid, Oid, int, unsigned int, BlockNumber);
 
 	probe buffer__checkpoint__start(int);
 	probe buffer__checkpoint__sync__start();
diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h
index fef5bc5382b..e9d8f3e6810 100644
--- a/src/include/storage/buf_internals.h
+++ b/src/include/storage/buf_internals.h
@@ -513,6 +513,33 @@ ResourceOwnerForgetBufferIO(ResourceOwner owner, Buffer buffer)
 	ResourceOwnerForget(owner, Int32GetDatum(buffer), &buffer_io_resowner_desc);
 }
 
+/*
+ * Used to write out multiple blocks at a time in a combined IO. bufdescs
+ * contains buffer descriptors for buffers containing adjacent blocks of the
+ * same fork of the same relation.
+ */
+typedef struct BufferWriteBatch
+{
+	ForkNumber	forkno;
+	SMgrRelation reln;
+
+	/*
+	 * The BlockNumber of the first block in the run of contiguous blocks to
+	 * be written out as a single IO.
+	 */
+	BlockNumber start;
+
+	/*
+	 * While assembling the buffers, we keep track of the maximum LSN so that
+	 * we can flush WAL through this LSN before flushing the buffers.
+	 */
+	XLogRecPtr	max_lsn;
+
+	/* The number of valid buffers in bufdescs */
+	uint32		n;
+	BufferDesc *bufdescs[MAX_IO_COMBINE_LIMIT];
+} BufferWriteBatch;
+
 /*
  * Internal buffer management routines
  */
@@ -521,6 +548,7 @@ extern void WritebackContextInit(WritebackContext *context, int *max_pending);
 extern void IssuePendingWritebacks(WritebackContext *wb_context, IOContext io_context);
 extern void ScheduleBufferTagForWriteback(WritebackContext *wb_context,
 										  IOContext io_context, BufferTag *tag);
+extern void FlushBufferBatch(BufferWriteBatch *batch, IOContext io_context);
 
 extern void TrackNewBufferPin(Buffer buf);
 extern bool BufferNeedsWALFlush(BufferDesc *bufdesc, bool exclusive_locked);
@@ -536,6 +564,8 @@ extern bool StrategySupportsEagerFlush(BufferAccessStrategy strategy);
 extern Buffer StrategyNextBuffer(BufferAccessStrategy strategy,
 								 int *cursor);
 extern int	StrategyGetCurrentIndex(BufferAccessStrategy strategy);
+extern void CompleteWriteBatchIO(BufferWriteBatch *batch, IOContext io_context,
+								 WritebackContext *wb_context);
 extern IOContext IOContextForStrategy(BufferAccessStrategy strategy);
 extern BufferDesc *StrategyGetBuffer(BufferAccessStrategy strategy,
 									 uint32 *buf_state, bool *from_ring);
diff --git a/src/include/storage/bufpage.h b/src/include/storage/bufpage.h
index ae3725b3b81..baadfc6c313 100644
--- a/src/include/storage/bufpage.h
+++ b/src/include/storage/bufpage.h
@@ -506,5 +506,7 @@ extern bool PageIndexTupleOverwrite(Page page, OffsetNumber offnum,
 									const void *newtup, Size newsize);
 extern char *PageSetChecksumCopy(Page page, BlockNumber blkno);
 extern void PageSetChecksumInplace(Page page, BlockNumber blkno);
+extern void PageSetBatchChecksumInplace(Page *pages, const BlockNumber *blknos,
+										uint32 length);
 
 #endif							/* BUFPAGE_H */
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 14dec2d49c1..bfdc6689fbf 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -359,6 +359,7 @@ BufferManagerRelation
 BufferStrategyControl
 BufferTag
 BufferUsage
+BufferWriteBatch
 BuildAccumulator
 BuiltinScript
 BulkInsertState
-- 
2.43.0

