From bd214faa0cf0d9574c73d7851d1c0e6fd54bb429 Mon Sep 17 00:00:00 2001
From: Yura Sokolov <y.sokolov@postgrespro.ru>
Date: Mon, 27 Jun 2022 17:24:00 +0300
Subject: [PATCH] bufmgr: do not acquire two partition locks.

Acquiring two partition locks leads to complex dependency chain that
hurts at high concurrency level.

There is no need to hold both locks simultaneously. Placeholder entry
could be inserted first both to reserve place in buffer table and to
inform other backends we're going to actually allocate this buffer.

Then other backends could wait on ConditionVariable associated with
backend doing actual work.

Buffer's tag backend is working on is saved into static variable, so
if backend is interrupted for some reason, it will remove placeholder
and wake up waiting backends.

Some ConditionVariable improvements are maid for performance reason:

- allow to skip ConditionVariableCancelSleep(void) with using new
  ConditionVariableSleepOnce(cv).
  CVSleepOnce could be not used in loop, since it doesn't re-enter
  process back into CV. Therefore it has prerequisite of already made
  CVPrepareToSleep(cv).

  It is save to use CVSleepOnce in our case since we're going to retry
  with fullblown LWLockAcquire+BufferLookup, and most probably it will
  succeed. Using CVSleepOnce save's one interaction with CV's spinlock,
  which is quite huge deal on contention.

- fetch and wakeup processes with ConditionVariableBroadcastFast(cv).
  Iterating one-by-one makes a lot of synchronized atomic writes on
  CV's spin lock, which harms performance significantly.
  And since awaking processes in batch doesn't harm correctness, lets
  do it.

---
 src/backend/access/transam/xlogprefetcher.c   |  10 +
 src/backend/storage/buffer/buf_table.c        |  62 +++--
 src/backend/storage/buffer/bufmgr.c           | 214 ++++++++++--------
 src/backend/storage/buffer/freelist.c         |  13 +-
 src/backend/storage/lmgr/condition_variable.c | 122 +++++++++-
 src/backend/utils/activity/wait_event.c       |   3 +
 src/include/storage/buf_internals.h           |   3 +-
 src/include/storage/bufmgr.h                  |   2 +
 src/include/storage/condition_variable.h      |   2 +
 src/include/utils/wait_event.h                |   1 +
 10 files changed, 312 insertions(+), 120 deletions(-)

diff --git a/src/backend/access/transam/xlogprefetcher.c b/src/backend/access/transam/xlogprefetcher.c
index 959e4094667..84274e60f97 100644
--- a/src/backend/access/transam/xlogprefetcher.c
+++ b/src/backend/access/transam/xlogprefetcher.c
@@ -782,6 +782,16 @@ XLogPrefetcherNextBlock(uintptr_t pgsr_private, XLogRecPtr *lsn)
 				block->prefetch_buffer = InvalidBuffer;
 				return LRQ_NEXT_IO;
 			}
+			else if (result.concurrent_io)
+			{
+				/*
+				 * Could it happen at all, i.e. could be there concurrent
+				 * backend trying to load same page? Anyway, try to do
+				 * something meaningful.
+				 */
+				block->prefetch_buffer = InvalidBuffer;
+				return LRQ_NEXT_NO_IO;
+			}
 			else
 			{
 				/*
diff --git a/src/backend/storage/buffer/buf_table.c b/src/backend/storage/buffer/buf_table.c
index dc439940faa..81d27a3b8d1 100644
--- a/src/backend/storage/buffer/buf_table.c
+++ b/src/backend/storage/buffer/buf_table.c
@@ -23,15 +23,17 @@
 
 #include "storage/buf_internals.h"
 #include "storage/bufmgr.h"
+#include "miscadmin.h"
 
 /* entry for buffer lookup hashtable */
 typedef struct
 {
 	BufferTag	key;			/* Tag of a disk page */
-	int			id;				/* Associated buffer ID */
+	volatile int id;			/* Associated buffer ID */
 } BufferLookupEnt;
 
 static HTAB *SharedBufHash;
+ConditionVariableMinimallyPadded *BufferInsertionCVArray;
 
 
 /*
@@ -41,7 +43,29 @@ static HTAB *SharedBufHash;
 Size
 BufTableShmemSize(int size)
 {
-	return hash_estimate_size(size, sizeof(BufferLookupEnt));
+	Size		sz;
+
+	/*
+	 * BufferAlloc inserts new buffer entry before deleting the old. That is
+	 * why there is a need in additional free entry for every backend.
+	 *
+	 * Also size could not be less than NUM_BUFFER_PARTITIONS.
+	 *
+	 * And since get_hash_entry is not very  inefficiency of dynahash's
+	 * get_hash_entry, it is better to have more spare entries, so we use both
+	 * MaxBackends and NUM_BUFFER_PARTITIONS.
+	 */
+	size += MaxBackends + NUM_BUFFER_PARTITIONS;
+	sz = hash_estimate_size(size, sizeof(BufferLookupEnt));
+
+	/*
+	 * Every backend should have associated ConditionVariable so we could map
+	 * them 1:1 without resolving collisions. And additional one is allocated
+	 * for startup process (xlog player), which have MyBackendId == -1.
+	 */
+	sz = add_size(sz, mul_size(MaxBackends + 1,
+							   sizeof(ConditionVariableMinimallyPadded)));
+	return sz;
 }
 
 /*
@@ -52,6 +76,10 @@ void
 InitBufTable(int size)
 {
 	HASHCTL		info;
+	bool		found;
+
+	/* see comments in BufTableShmemSize */
+	size += MaxBackends + NUM_BUFFER_PARTITIONS;
 
 	/* assume no locking is needed yet */
 
@@ -63,7 +91,18 @@ InitBufTable(int size)
 	SharedBufHash = ShmemInitHash("Shared Buffer Lookup Table",
 								  size, size,
 								  &info,
-								  HASH_ELEM | HASH_BLOBS | HASH_PARTITION);
+								  HASH_ELEM | HASH_BLOBS | HASH_PARTITION |
+								  HASH_FIXED_SIZE);
+	BufferInsertionCVArray = (ConditionVariableMinimallyPadded *)
+		ShmemInitStruct("Shared Buffer Backend Insertion CV",
+						(MaxBackends + 1) *
+						sizeof(ConditionVariableMinimallyPadded),
+						&found);
+	if (!found)
+	{
+		for (int i = 0; i < MaxBackends + 1; i++)
+			ConditionVariableInit(&BufferInsertionCVArray[i].cv);
+	}
 }
 
 /*
@@ -110,18 +149,17 @@ BufTableLookup(BufferTag *tagPtr, uint32 hashcode)
  *		Insert a hashtable entry for given tag and buffer ID,
  *		unless an entry already exists for that tag
  *
- * Returns -1 on successful insertion.  If a conflicting entry exists
- * already, returns the buffer ID in that entry.
+ * Returns pointer to volatile int holding buffer index.
+ * If table entry were just inserted, index is filled with -1.
  *
  * Caller must hold exclusive lock on BufMappingLock for tag's partition
  */
-int
-BufTableInsert(BufferTag *tagPtr, uint32 hashcode, int buf_id)
+volatile int *
+BufTableInsert(BufferTag *tagPtr, uint32 hashcode)
 {
 	BufferLookupEnt *result;
 	bool		found;
 
-	Assert(buf_id >= 0);		/* -1 is reserved for not-in-table */
 	Assert(tagPtr->blockNum != P_NEW);	/* invalid tag */
 
 	result = (BufferLookupEnt *)
@@ -131,12 +169,10 @@ BufTableInsert(BufferTag *tagPtr, uint32 hashcode, int buf_id)
 									HASH_ENTER,
 									&found);
 
-	if (found)					/* found something already in the table */
-		return result->id;
-
-	result->id = buf_id;
+	if (!found)
+		result->id = -1;
 
-	return -1;
+	return &result->id;
 }
 
 /*
diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c
index ae13011d275..e5d79737f2e 100644
--- a/src/backend/storage/buffer/bufmgr.c
+++ b/src/backend/storage/buffer/bufmgr.c
@@ -166,6 +166,9 @@ static bool IsForInput;
 /* local state for LockBufferForCleanup */
 static BufferDesc *PinCountWaitBuf = NULL;
 
+/* tag of speculatively inserted placeholder */
+static BufferTag PlaceholderTag = {{0, 0, 0}, 0, 0};
+
 /*
  * Backend-Private refcount management:
  *
@@ -506,7 +509,7 @@ PrefetchSharedBuffer(SMgrRelation smgr_reln,
 					 ForkNumber forkNum,
 					 BlockNumber blockNum)
 {
-	PrefetchBufferResult result = {InvalidBuffer, false};
+	PrefetchBufferResult result = {InvalidBuffer, false, false};
 	BufferTag	newTag;			/* identity of requested block */
 	uint32		newHash;		/* hash value for newTag */
 	LWLock	   *newPartitionLock;	/* buffer partition lock for it */
@@ -528,7 +531,7 @@ PrefetchSharedBuffer(SMgrRelation smgr_reln,
 	LWLockRelease(newPartitionLock);
 
 	/* If not in buffers, initiate prefetch */
-	if (buf_id < 0)
+	if (buf_id == -1)
 	{
 #ifdef USE_PREFETCH
 		/*
@@ -539,7 +542,7 @@ PrefetchSharedBuffer(SMgrRelation smgr_reln,
 			result.initiated_io = true;
 #endif							/* USE_PREFETCH */
 	}
-	else
+	else if (buf_id >= 0)
 	{
 		/*
 		 * Report the buffer it was in at that time.  The caller may be able
@@ -548,6 +551,11 @@ PrefetchSharedBuffer(SMgrRelation smgr_reln,
 		 */
 		result.recent_buffer = buf_id + 1;
 	}
+	else
+	{
+		/* Other backend is in process of insertion. */
+		result.concurrent_io = true;
+	}
 
 	/*
 	 * If the block *is* in buffers, we do nothing.  This is not really ideal:
@@ -1114,13 +1122,17 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
 	BufferTag	newTag;			/* identity of requested block */
 	uint32		newHash;		/* hash value for newTag */
 	LWLock	   *newPartitionLock;	/* buffer partition lock for it */
+	ConditionVariable *insertionCV;
 	BufferTag	oldTag;			/* previous identity of selected buffer */
 	uint32		oldHash;		/* hash value for oldTag */
 	LWLock	   *oldPartitionLock;	/* buffer partition lock for it */
 	uint32		oldFlags;
 	int			buf_id;
+	int			backId;
+	volatile int *buf_id_p;
 	BufferDesc *buf;
 	bool		valid;
+	bool		wait;
 	uint32		buf_state;
 
 	/* create a tag so we can lookup the buffer */
@@ -1131,6 +1143,7 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
 	newPartitionLock = BufMappingPartitionLock(newHash);
 
 	/* see if the block is in the buffer pool already */
+retry:
 	LWLockAcquire(newPartitionLock, LW_SHARED);
 	buf_id = BufTableLookup(&newTag, newHash);
 	if (buf_id >= 0)
@@ -1173,9 +1186,58 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
 
 	/*
 	 * Didn't find it in the buffer pool.  We'll have to initialize a new
-	 * buffer.  Remember to unlock the mapping lock while doing the work.
+	 * buffer.
+	 *
+	 * First, we insert placeholder to indicate we're working on (or to find
+	 * someone else is working). So re-lock partition in exclusive mode.
 	 */
 	LWLockRelease(newPartitionLock);
+	LWLockAcquire(newPartitionLock, LW_EXCLUSIVE);
+
+	buf_id_p = BufTableInsert(&newTag, newHash);
+	buf_id = *buf_id_p;
+
+	/* During startup MyBackendId == -1, so workaround. */
+	backId = MyBackendId > 0 ? MyBackendId - 1 : MaxBackends;
+
+	if (buf_id == -1)
+	{
+		/* Ok, we are first, who tried this buffer. Mark with our backend id. */
+		*buf_id_p = -2 - backId;
+		/* And remember we're inserting it for cleanup procs */
+		PlaceholderTag = newTag;
+	}
+	else if (buf_id < -1)
+	{
+		/* Someone else is trying to insert this buffer. We should wait him. */
+		insertionCV = &BufferInsertionCVArray[-2 - buf_id].cv;
+		ConditionVariablePrepareToSleep(insertionCV);
+
+		/*
+		 * buf_id_p is finally written further without holding partition lock.
+		 * So backend which inserted it could already call CVBroadcast before
+		 * we enqueue ourselves into CV. Therefore, we must recheck buf_id_p
+		 * content now.
+		 *
+		 * buf_id_p is still valid pointer since it could be deleted/reused
+		 * only with exclusive partition lock.
+		 *
+		 * We are relying on 32bit atomicity here as in several other places.
+		 */
+		wait = *buf_id_p == buf_id;
+	}
+	LWLockRelease(newPartitionLock);
+
+	if (buf_id < -1)
+	{
+		if (wait)
+			ConditionVariableSleepOnce(insertionCV, WAIT_EVENT_BUFFER_INSERT);
+		else
+			ConditionVariableCancelSleep();
+	}
+
+	if (buf_id != -1)
+		goto retry;
 
 	/* Loop here in case we have to try another victim buffer */
 	for (;;)
@@ -1283,7 +1345,10 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
 
 		/*
 		 * To change the association of a valid buffer, we'll need to have
-		 * exclusive lock on both the old and new mapping partitions.
+		 * exclusive lock on old mapping partition.
+		 *
+		 * Note: we don't need to have the lock on new partition since we're
+		 * effectively locking placeholder entry.
 		 */
 		if (oldFlags & BM_TAG_VALID)
 		{
@@ -1296,93 +1361,16 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
 			oldHash = BufTableHashCode(&oldTag);
 			oldPartitionLock = BufMappingPartitionLock(oldHash);
 
-			/*
-			 * Must lock the lower-numbered partition first to avoid
-			 * deadlocks.
-			 */
-			if (oldPartitionLock < newPartitionLock)
-			{
-				LWLockAcquire(oldPartitionLock, LW_EXCLUSIVE);
-				LWLockAcquire(newPartitionLock, LW_EXCLUSIVE);
-			}
-			else if (oldPartitionLock > newPartitionLock)
-			{
-				LWLockAcquire(newPartitionLock, LW_EXCLUSIVE);
-				LWLockAcquire(oldPartitionLock, LW_EXCLUSIVE);
-			}
-			else
-			{
-				/* only one partition, only one lock */
-				LWLockAcquire(newPartitionLock, LW_EXCLUSIVE);
-			}
+			LWLockAcquire(oldPartitionLock, LW_EXCLUSIVE);
 		}
 		else
 		{
-			/* if it wasn't valid, we need only the new partition */
-			LWLockAcquire(newPartitionLock, LW_EXCLUSIVE);
 			/* remember we have no old-partition lock or tag */
 			oldPartitionLock = NULL;
 			/* keep the compiler quiet about uninitialized variables */
 			oldHash = 0;
 		}
 
-		/*
-		 * Try to make a hashtable entry for the buffer under its new tag.
-		 * This could fail because while we were writing someone else
-		 * allocated another buffer for the same block we want to read in.
-		 * Note that we have not yet removed the hashtable entry for the old
-		 * tag.
-		 */
-		buf_id = BufTableInsert(&newTag, newHash, buf->buf_id);
-
-		if (buf_id >= 0)
-		{
-			/*
-			 * Got a collision. Someone has already done what we were about to
-			 * do. We'll just handle this as if it were found in the buffer
-			 * pool in the first place.  First, give up the buffer we were
-			 * planning to use.
-			 */
-			UnpinBuffer(buf, true);
-
-			/* Can give up that buffer's mapping partition lock now */
-			if (oldPartitionLock != NULL &&
-				oldPartitionLock != newPartitionLock)
-				LWLockRelease(oldPartitionLock);
-
-			/* remaining code should match code at top of routine */
-
-			buf = GetBufferDescriptor(buf_id);
-
-			valid = PinBuffer(buf, strategy);
-
-			/* Can release the mapping lock as soon as we've pinned it */
-			LWLockRelease(newPartitionLock);
-
-			*foundPtr = true;
-
-			if (!valid)
-			{
-				/*
-				 * We can only get here if (a) someone else is still reading
-				 * in the page, or (b) a previous read attempt failed.  We
-				 * have to wait for any active read attempt to finish, and
-				 * then set up our own read attempt if the page is still not
-				 * BM_VALID.  StartBufferIO does it all.
-				 */
-				if (StartBufferIO(buf, true))
-				{
-					/*
-					 * If we get here, previous attempts to read the buffer
-					 * must have failed ... but we shall bravely try again.
-					 */
-					*foundPtr = false;
-				}
-			}
-
-			return buf;
-		}
-
 		/*
 		 * Need to lock the buffer header too in order to change its tag.
 		 */
@@ -1390,20 +1378,16 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
 
 		/*
 		 * Somebody could have pinned or re-dirtied the buffer while we were
-		 * doing the I/O and making the new hashtable entry.  If so, we can't
-		 * recycle this buffer; we must undo everything we've done and start
-		 * over with a new victim buffer.
+		 * doing the I/O.  If so, we can't recycle this buffer; we must undo
+		 * everything we've done and start over with a new victim buffer.
 		 */
 		oldFlags = buf_state & BUF_FLAG_MASK;
 		if (BUF_STATE_GET_REFCOUNT(buf_state) == 1 && !(oldFlags & BM_DIRTY))
 			break;
 
 		UnlockBufHdr(buf, buf_state);
-		BufTableDelete(&newTag, newHash);
-		if (oldPartitionLock != NULL &&
-			oldPartitionLock != newPartitionLock)
+		if (oldPartitionLock != NULL)
 			LWLockRelease(oldPartitionLock);
-		LWLockRelease(newPartitionLock);
 		UnpinBuffer(buf, true);
 	}
 
@@ -1429,16 +1413,24 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
 	else
 		buf_state |= BM_TAG_VALID | BUF_USAGECOUNT_ONE;
 
+	/*
+	 * need to write buf_id_p before unlock buffer header since BM_TAG_VALID
+	 * means BufTable has an entry pointing to this buffer header.
+	 */
+	*buf_id_p = buf->buf_id;
 	UnlockBufHdr(buf, buf_state);
 
+	/* We're done. Forget about placeholder */
+	CLEAR_BUFFERTAG(PlaceholderTag);
+
 	if (oldPartitionLock != NULL)
 	{
 		BufTableDelete(&oldTag, oldHash);
-		if (oldPartitionLock != newPartitionLock)
-			LWLockRelease(oldPartitionLock);
+		LWLockRelease(oldPartitionLock);
 	}
 
-	LWLockRelease(newPartitionLock);
+	insertionCV = &BufferInsertionCVArray[backId].cv;
+	ConditionVariableBroadcastFast(insertionCV);
 
 	/*
 	 * Buffer contents are currently invalid.  Try to obtain the right to
@@ -2576,16 +2568,50 @@ SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context)
 	return result | BUF_WRITTEN;
 }
 
+static void
+BufferCleanupPlaceholder(void)
+{
+	uint32		hash;			/* hash value for newTag */
+	LWLock	   *partitionLock;	/* buffer partition lock for it */
+	int			backId;
+
+	if (likely(PlaceholderTag.rnode.relNode == InvalidOid))
+		return;
+
+	hash = BufTableHashCode(&PlaceholderTag);
+	partitionLock = BufMappingPartitionLock(hash);
+
+	/* During startup MyBackendId == -1, so workaround. */
+	backId = MyBackendId > 0 ? MyBackendId - 1 : MaxBackends;
+	LWLockAcquire(partitionLock, LW_EXCLUSIVE);
+
+	/*
+	 * Just last sanity check. should always be true. I don't know how to
+	 * trigger this in tests, so no Assert here.
+	 */
+	if (BufTableLookup(&PlaceholderTag, hash) == -2 - backId)
+		BufTableDelete(&PlaceholderTag, hash);
+	LWLockRelease(partitionLock);
+
+	ConditionVariableBroadcastFast(&BufferInsertionCVArray[backId].cv);
+
+	CLEAR_BUFFERTAG(PlaceholderTag);
+}
+
 /*
  *		AtEOXact_Buffers - clean up at end of transaction.
  *
  *		As of PostgreSQL 8.0, buffer pins should get released by the
  *		ResourceOwner mechanism.  This routine is just a debugging
  *		cross-check that no pins remain.
+ *
+ *		Except for sanity check process didn't fall during buffer
+ *		insertion. Then it should delete placeholder.
  */
 void
 AtEOXact_Buffers(bool isCommit)
 {
+	BufferCleanupPlaceholder();
 	CheckForBufferLeaks();
 
 	AtEOXact_LocalBuffers(isCommit);
@@ -2628,6 +2654,7 @@ InitBufferPoolAccess(void)
 static void
 AtProcExit_Buffers(int code, Datum arg)
 {
+	BufferCleanupPlaceholder();
 	AbortBufferIO();
 	UnlockBuffers();
 
@@ -3367,6 +3394,7 @@ FindAndDropRelFileNodeBuffers(RelFileNode rnode, ForkNumber forkNum,
 		buf_id = BufTableLookup(&bufTag, bufHash);
 		LWLockRelease(bufPartitionLock);
 
+		Assert(buf_id >= -1);
 		if (buf_id < 0)
 			continue;
 
diff --git a/src/backend/storage/buffer/freelist.c b/src/backend/storage/buffer/freelist.c
index 990e081aaec..aa440e077f3 100644
--- a/src/backend/storage/buffer/freelist.c
+++ b/src/backend/storage/buffer/freelist.c
@@ -454,8 +454,8 @@ StrategyShmemSize(void)
 {
 	Size		size = 0;
 
-	/* size of lookup hash table ... see comment in StrategyInitialize */
-	size = add_size(size, BufTableShmemSize(NBuffers + NUM_BUFFER_PARTITIONS));
+	/* size of lookup hash table */
+	size = add_size(size, BufTableShmemSize(NBuffers));
 
 	/* size of the shared replacement strategy control block */
 	size = add_size(size, MAXALIGN(sizeof(BufferStrategyControl)));
@@ -477,15 +477,8 @@ StrategyInitialize(bool init)
 
 	/*
 	 * Initialize the shared buffer lookup hashtable.
-	 *
-	 * Since we can't tolerate running out of lookup table entries, we must be
-	 * sure to specify an adequate table size here.  The maximum steady-state
-	 * usage is of course NBuffers entries, but BufferAlloc() tries to insert
-	 * a new entry before deleting the old.  In principle this could be
-	 * happening in each partition concurrently, so we could need as many as
-	 * NBuffers + NUM_BUFFER_PARTITIONS entries.
 	 */
-	InitBufTable(NBuffers + NUM_BUFFER_PARTITIONS);
+	InitBufTable(NBuffers);
 
 	/*
 	 * Get or create the shared strategy control block
diff --git a/src/backend/storage/lmgr/condition_variable.c b/src/backend/storage/lmgr/condition_variable.c
index de65dac3ae0..60f63762a31 100644
--- a/src/backend/storage/lmgr/condition_variable.c
+++ b/src/backend/storage/lmgr/condition_variable.c
@@ -30,6 +30,9 @@
 /* Initially, we are not prepared to sleep on any condition variable. */
 static ConditionVariable *cv_sleep_target = NULL;
 
+static bool ConditionVariableTimedSleepImpl(ConditionVariable *cv, long timeout,
+											uint32 wait_event_info, bool once);
+
 /*
  * Initialize a condition variable.
  */
@@ -97,8 +100,15 @@ ConditionVariablePrepareToSleep(ConditionVariable *cv)
 void
 ConditionVariableSleep(ConditionVariable *cv, uint32 wait_event_info)
 {
-	(void) ConditionVariableTimedSleep(cv, -1 /* no timeout */ ,
-									   wait_event_info);
+	(void) ConditionVariableTimedSleepImpl(cv, -1 /* no timeout */ ,
+										   wait_event_info, false);
+}
+
+void
+ConditionVariableSleepOnce(ConditionVariable *cv, uint32 wait_event_info)
+{
+	(void) ConditionVariableTimedSleepImpl(cv, -1 /* no timeout */ ,
+										   wait_event_info, true);
 }
 
 /*
@@ -111,6 +121,20 @@ ConditionVariableSleep(ConditionVariable *cv, uint32 wait_event_info)
 bool
 ConditionVariableTimedSleep(ConditionVariable *cv, long timeout,
 							uint32 wait_event_info)
+{
+	return ConditionVariableTimedSleepImpl(cv, timeout, wait_event_info, false);
+}
+
+/*
+ * Wait for a condition variable to be signaled or a timeout to be reached.
+ *
+ * Returns true when timeout expires, otherwise returns false.
+ *
+ * See ConditionVariableSleep() for general usage.
+ */
+static bool
+ConditionVariableTimedSleepImpl(ConditionVariable *cv, long timeout,
+								uint32 wait_event_info, bool once)
 {
 	long		cur_timeout = -1;
 	instr_time	start_time;
@@ -134,6 +158,9 @@ ConditionVariableTimedSleep(ConditionVariable *cv, long timeout,
 	 */
 	if (cv_sleep_target != cv)
 	{
+		if (once)
+			elog(FATAL, "ConditionVariable could be waited once only if it is "
+				 "prepared to sleep.");
 		ConditionVariablePrepareToSleep(cv);
 		return false;
 	}
@@ -184,7 +211,8 @@ ConditionVariableTimedSleep(ConditionVariable *cv, long timeout,
 		if (!proclist_contains(&cv->wakeup, MyProc->pgprocno, cvWaitLink))
 		{
 			done = true;
-			proclist_push_tail(&cv->wakeup, MyProc->pgprocno, cvWaitLink);
+			if (!once)
+				proclist_push_tail(&cv->wakeup, MyProc->pgprocno, cvWaitLink);
 		}
 		SpinLockRelease(&cv->mutex);
 
@@ -199,7 +227,11 @@ ConditionVariableTimedSleep(ConditionVariable *cv, long timeout,
 
 		/* We were signaled, so return */
 		if (done)
+		{
+			if (once && cv == cv_sleep_target)
+				cv_sleep_target = NULL;
 			return false;
+		}
 
 		/* If we're not done, update cur_timeout for next iteration */
 		if (timeout >= 0)
@@ -362,3 +394,87 @@ ConditionVariableBroadcast(ConditionVariable *cv)
 			SetLatch(&proc->procLatch);
 	}
 }
+
+void
+ConditionVariableBroadcastFast(ConditionVariable *cv)
+{
+	int			pgprocno = MyProc->pgprocno;
+	bool		have_sentinel = false;
+#define BATCH_SIZE 16
+	PGPROC	   *proc = NULL;
+	PGPROC	   *procs[BATCH_SIZE] = {NULL};
+	int			nprocs = 0;
+
+	/*
+	 * In some use-cases, it is common for awakened processes to immediately
+	 * re-queue themselves.  If we just naively try to reduce the wakeup list
+	 * to empty, we'll get into a potentially-indefinite loop against such a
+	 * process.  The semantics we really want are just to be sure that we have
+	 * wakened all processes that were in the list at entry.  We can use our
+	 * own cvWaitLink as a sentinel to detect when we've finished.
+	 *
+	 * A seeming flaw in this approach is that someone else might signal the
+	 * CV and in doing so remove our sentinel entry.  But that's fine: since
+	 * CV waiters are always added and removed in order, that must mean that
+	 * every previous waiter has been wakened, so we're done.  We'll get an
+	 * extra "set" on our latch from the someone else's signal, which is
+	 * slightly inefficient but harmless.
+	 *
+	 * We can't insert our cvWaitLink as a sentinel if it's already in use in
+	 * some other proclist.  While that's not expected to be true for typical
+	 * uses of this function, we can deal with it by simply canceling any
+	 * prepared CV sleep.  The next call to ConditionVariableSleep will take
+	 * care of re-establishing the lost state.
+	 */
+	if (cv_sleep_target != NULL)
+		ConditionVariableCancelSleep();
+
+	do
+	{
+		/*
+		 * Fetch processes from proclist in batches.
+		 *
+		 * If there were less entries than batch size on first iteration, we
+		 * will done immediately. Otherwise we insert sentinel entry do detect
+		 * part of list we are responsible for.
+		 *
+		 * Notice that if someone else removes our sentinel, we will waken up
+		 * to BATCH_SIZE of additional processes before exiting.  That's
+		 * intentional, because if someone else signals the CV, they may be
+		 * intending to waken some third process that added itself to the list
+		 * after we added the sentinel.  Better to give a spurious wakeup
+		 * (which should be harmless beyond wasting some cycles) than to lose
+		 * a wakeup.
+		 */
+		SpinLockAcquire(&cv->mutex);
+		if (!have_sentinel)
+			/* While we're here, let's assert we're not in the list. */
+			Assert(!proclist_contains(&cv->wakeup, pgprocno, cvWaitLink));
+
+		while (nprocs < BATCH_SIZE && !proclist_is_empty(&cv->wakeup))
+		{
+			proc = proclist_pop_head_node(&cv->wakeup, cvWaitLink);
+			if (proc == MyProc)
+				break;
+			procs[nprocs++] = proc;
+		}
+
+		if (!have_sentinel && !proclist_is_empty(&cv->wakeup))
+		{
+			proclist_push_tail(&cv->wakeup, pgprocno, cvWaitLink);
+			have_sentinel = true;
+		}
+		else if (have_sentinel)
+		{
+			have_sentinel = proclist_contains(&cv->wakeup, pgprocno, cvWaitLink);
+		}
+		SpinLockRelease(&cv->mutex);
+
+		/* Awaken waiters batch, if there were some. */
+		while (nprocs > 0)
+		{
+			proc = procs[--nprocs];
+			SetLatch(&proc->procLatch);
+		}
+	} while (have_sentinel);
+}
diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c
index 87c15b9c6f3..e121a301409 100644
--- a/src/backend/utils/activity/wait_event.c
+++ b/src/backend/utils/activity/wait_event.c
@@ -331,6 +331,9 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_BTREE_PAGE:
 			event_name = "BtreePage";
 			break;
+		case WAIT_EVENT_BUFFER_INSERT:
+			event_name = "BufferInsert";
+			break;
 		case WAIT_EVENT_BUFFER_IO:
 			event_name = "BufferIO";
 			break;
diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h
index a17e7b28a53..5f2d73c6773 100644
--- a/src/include/storage/buf_internals.h
+++ b/src/include/storage/buf_internals.h
@@ -231,6 +231,7 @@ typedef union BufferDescPadded
 	((LWLock*) (&(bdesc)->content_lock))
 
 extern PGDLLIMPORT ConditionVariableMinimallyPadded *BufferIOCVArray;
+extern PGDLLIMPORT ConditionVariableMinimallyPadded *BufferInsertionCVArray;
 
 /*
  * The freeNext field is either the index of the next freelist entry,
@@ -327,7 +328,7 @@ extern Size BufTableShmemSize(int size);
 extern void InitBufTable(int size);
 extern uint32 BufTableHashCode(BufferTag *tagPtr);
 extern int	BufTableLookup(BufferTag *tagPtr, uint32 hashcode);
-extern int	BufTableInsert(BufferTag *tagPtr, uint32 hashcode, int buf_id);
+extern volatile int *BufTableInsert(BufferTag *tagPtr, uint32 hashcode);
 extern void BufTableDelete(BufferTag *tagPtr, uint32 hashcode);
 
 /* localbuf.c */
diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h
index 58391406f65..a11d04290f1 100644
--- a/src/include/storage/bufmgr.h
+++ b/src/include/storage/bufmgr.h
@@ -53,6 +53,8 @@ typedef struct PrefetchBufferResult
 {
 	Buffer		recent_buffer;	/* If valid, a hit (recheck needed!) */
 	bool		initiated_io;	/* If true, a miss resulting in async I/O */
+	bool		concurrent_io;	/* If true, other backend is trying to load
+								 * page at the moment */
 } PrefetchBufferResult;
 
 /* forward declared, to avoid having to expose buf_internals.h here */
diff --git a/src/include/storage/condition_variable.h b/src/include/storage/condition_variable.h
index e89175ebd5c..76a7330c132 100644
--- a/src/include/storage/condition_variable.h
+++ b/src/include/storage/condition_variable.h
@@ -57,6 +57,7 @@ extern void ConditionVariableSleep(ConditionVariable *cv, uint32 wait_event_info
 extern bool ConditionVariableTimedSleep(ConditionVariable *cv, long timeout,
 										uint32 wait_event_info);
 extern void ConditionVariableCancelSleep(void);
+extern void ConditionVariableSleepOnce(ConditionVariable *cv, uint32 wait_event_info);
 
 /*
  * Optionally, ConditionVariablePrepareToSleep can be called before entering
@@ -69,5 +70,6 @@ extern void ConditionVariablePrepareToSleep(ConditionVariable *cv);
 /* Wake up a single waiter (via signal) or all waiters (via broadcast). */
 extern void ConditionVariableSignal(ConditionVariable *cv);
 extern void ConditionVariableBroadcast(ConditionVariable *cv);
+extern void ConditionVariableBroadcastFast(ConditionVariable *cv);
 
 #endif							/* CONDITION_VARIABLE_H */
diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h
index b578e2ec757..4120c45de41 100644
--- a/src/include/utils/wait_event.h
+++ b/src/include/utils/wait_event.h
@@ -86,6 +86,7 @@ typedef enum
 	WAIT_EVENT_BGWORKER_SHUTDOWN,
 	WAIT_EVENT_BGWORKER_STARTUP,
 	WAIT_EVENT_BTREE_PAGE,
+	WAIT_EVENT_BUFFER_INSERT,
 	WAIT_EVENT_BUFFER_IO,
 	WAIT_EVENT_CHECKPOINT_DONE,
 	WAIT_EVENT_CHECKPOINT_START,
-- 
2.36.1

