From 89b7feb30cb70898595366c0203d420c9502c7b6 Mon Sep 17 00:00:00 2001
From: Vignesh C <vignesh21@gmail.com>
Date: Wed, 29 May 2024 14:55:49 +0530
Subject: [PATCH v2] Improved latch handling between the logical replication
 launcher and worker processes.

The current implementation uses a single latch for multiple purposes:
a) attaching worker processes, b) handling worker process exits, and
c) creating subscriptions. This overlap causes issues in concurrent scenarios,
such as when the launcher starts a new apply worker and is waiting for it to
attach, while simultaneously a new subscription creation request arrives.

This commit fixes the problem by introducing a new wakeupLatch. This latch
will be set once the logical replication processes (apply worker,
parallel apply worker, or tablesync worker) are started. Consequently,
bgw_notify_pid is no longer required, as wakeupLatch will handle notifications
when a logical replication worker process attaches and will signal the
launcher or apply worker upon process exit from logicalrep_worker_onexit.
---
 .../replication/logical/applyparallelworker.c |  1 +
 src/backend/replication/logical/launcher.c    | 93 +++++++++++--------
 src/backend/replication/logical/worker.c      |  1 +
 src/include/replication/worker_internal.h     |  7 ++
 4 files changed, 62 insertions(+), 40 deletions(-)

diff --git a/src/backend/replication/logical/applyparallelworker.c b/src/backend/replication/logical/applyparallelworker.c
index e7f7d4c5e4..2fb4d63ebc 100644
--- a/src/backend/replication/logical/applyparallelworker.c
+++ b/src/backend/replication/logical/applyparallelworker.c
@@ -912,6 +912,7 @@ ParallelApplyWorkerMain(Datum main_arg)
 	 * the uninitialized memory queue.
 	 */
 	logicalrep_worker_attach(worker_slot);
+	SetLatch(&MyLogicalRepWorker->wakeupLatch);
 
 	/*
 	 * Register the shutdown callback after we are attached to the worker
diff --git a/src/backend/replication/logical/launcher.c b/src/backend/replication/logical/launcher.c
index c566d50a07..dea41b7ea1 100644
--- a/src/backend/replication/logical/launcher.c
+++ b/src/backend/replication/logical/launcher.c
@@ -171,6 +171,35 @@ get_subscription_list(void)
 	return res;
 }
 
+/*
+ * Wait for wakeupLatch to be set or a timeout.
+ */
+static void
+WaitForWakeupLatch(LogicalRepWorker *worker, uint32 wait_event_info)
+{
+	int			rc;
+
+	SpinLockAcquire(&worker->wakeupmutex);
+	OwnLatch(&worker->wakeupLatch);
+
+	rc = WaitLatch(&worker->wakeupLatch,
+					WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
+					1000L, wait_event_info);
+
+	if (rc & WL_LATCH_SET)
+	{
+		ResetLatch(&worker->wakeupLatch);
+		DisownLatch(&worker->wakeupLatch);
+		SpinLockRelease(&worker->wakeupmutex);
+		CHECK_FOR_INTERRUPTS();
+	}
+	else
+	{
+		DisownLatch(&worker->wakeupLatch);
+		SpinLockRelease(&worker->wakeupmutex);
+	}
+}
+
 /*
  * Wait for a background worker to start up and attach to the shmem context.
  *
@@ -185,7 +214,6 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker,
 							   BackgroundWorkerHandle *handle)
 {
 	BgwHandleStatus status;
-	int			rc;
 
 	for (;;)
 	{
@@ -217,19 +245,7 @@ WaitForReplicationWorkerAttach(LogicalRepWorker *worker,
 			return false;
 		}
 
-		/*
-		 * We need timeout because we generally don't get notified via latch
-		 * about the worker attach.  But we don't expect to have to wait long.
-		 */
-		rc = WaitLatch(MyLatch,
-					   WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
-					   10L, WAIT_EVENT_BGWORKER_STARTUP);
-
-		if (rc & WL_LATCH_SET)
-		{
-			ResetLatch(MyLatch);
-			CHECK_FOR_INTERRUPTS();
-		}
+		WaitForWakeupLatch(worker, WAIT_EVENT_BGWORKER_STARTUP);
 	}
 }
 
@@ -456,6 +472,12 @@ retry:
 	worker->reply_lsn = InvalidXLogRecPtr;
 	TIMESTAMP_NOBEGIN(worker->reply_time);
 
+	SpinLockAcquire(&worker->wakeupmutex);
+	OwnLatch(&worker->wakeupLatch);
+	ResetLatch(&worker->wakeupLatch);
+	DisownLatch(&worker->wakeupLatch);
+	SpinLockRelease(&worker->wakeupmutex);
+
 	/* Before releasing lock, remember generation for future identification. */
 	generation = worker->generation;
 
@@ -503,7 +525,7 @@ retry:
 	}
 
 	bgw.bgw_restart_time = BGW_NEVER_RESTART;
-	bgw.bgw_notify_pid = MyProcPid;
+	bgw.bgw_notify_pid = 0;
 	bgw.bgw_main_arg = Int32GetDatum(slot);
 
 	if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle))
@@ -548,20 +570,9 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo)
 	 */
 	while (worker->in_use && !worker->proc)
 	{
-		int			rc;
-
 		LWLockRelease(LogicalRepWorkerLock);
 
-		/* Wait a bit --- we don't expect to have to wait long. */
-		rc = WaitLatch(MyLatch,
-					   WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
-					   10L, WAIT_EVENT_BGWORKER_STARTUP);
-
-		if (rc & WL_LATCH_SET)
-		{
-			ResetLatch(MyLatch);
-			CHECK_FOR_INTERRUPTS();
-		}
+		WaitForWakeupLatch(worker, WAIT_EVENT_BGWORKER_STARTUP);
 
 		/* Recheck worker status. */
 		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
@@ -585,24 +596,13 @@ logicalrep_worker_stop_internal(LogicalRepWorker *worker, int signo)
 	/* ... and wait for it to die. */
 	for (;;)
 	{
-		int			rc;
-
 		/* is it gone? */
 		if (!worker->proc || worker->generation != generation)
 			break;
 
 		LWLockRelease(LogicalRepWorkerLock);
 
-		/* Wait a bit --- we don't expect to have to wait long. */
-		rc = WaitLatch(MyLatch,
-					   WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
-					   10L, WAIT_EVENT_BGWORKER_SHUTDOWN);
-
-		if (rc & WL_LATCH_SET)
-		{
-			ResetLatch(MyLatch);
-			CHECK_FOR_INTERRUPTS();
-		}
+		WaitForWakeupLatch(worker, WAIT_EVENT_BGWORKER_SHUTDOWN);
 
 		LWLockAcquire(LogicalRepWorkerLock, LW_SHARED);
 	}
@@ -823,6 +823,8 @@ logicalrep_launcher_onexit(int code, Datum arg)
 static void
 logicalrep_worker_onexit(int code, Datum arg)
 {
+	LogicalRepWorkerType type = MyLogicalRepWorker->type;
+
 	/* Disconnect gracefully from the remote side. */
 	if (LogRepWorkerWalRcvConn)
 		walrcv_disconnect(LogRepWorkerWalRcvConn);
@@ -843,7 +845,16 @@ logicalrep_worker_onexit(int code, Datum arg)
 	if (!InitializingApplyWorker)
 		LockReleaseAll(DEFAULT_LOCKMETHOD, true);
 
-	ApplyLauncherWakeup();
+	SetLatch(&MyLogicalRepWorker->wakeupLatch);
+
+	/*
+	 * Since we don't set bgw_notify_pid, send an exit signal to the parent
+	 * process.
+	 */
+	if (type == WORKERTYPE_APPLY)
+		ApplyLauncherWakeup();
+	else
+		logicalrep_worker_wakeup(MyLogicalRepWorker->subid, InvalidOid);
 }
 
 /*
@@ -982,6 +993,8 @@ ApplyLauncherShmemInit(void)
 
 			memset(worker, 0, sizeof(LogicalRepWorker));
 			SpinLockInit(&worker->relmutex);
+			InitSharedLatch(&worker->wakeupLatch);
+			SpinLockInit(&worker->wakeupmutex);
 		}
 	}
 }
diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c
index 0fb577d328..15cfe5693e 100644
--- a/src/backend/replication/logical/worker.c
+++ b/src/backend/replication/logical/worker.c
@@ -4737,6 +4737,7 @@ SetupApplyOrSyncWorker(int worker_slot)
 	pqsignal(SIGHUP, SignalHandlerForConfigReload);
 	pqsignal(SIGTERM, die);
 	BackgroundWorkerUnblockSignals();
+	SetLatch(&MyLogicalRepWorker->wakeupLatch);
 
 	/*
 	 * We don't currently need any ResourceOwner in a walreceiver process, but
diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h
index 9646261d7e..61f63dc1a3 100644
--- a/src/include/replication/worker_internal.h
+++ b/src/include/replication/worker_internal.h
@@ -77,6 +77,13 @@ typedef struct LogicalRepWorker
 	 */
 	FileSet    *stream_fileset;
 
+	/*
+	 * Used to wake up the launcher/apply worker after the logical replication
+	 * process is attached/exited.
+	 */
+	Latch		wakeupLatch;
+	slock_t		wakeupmutex;
+
 	/*
 	 * PID of leader apply worker if this slot is used for a parallel apply
 	 * worker, InvalidPid otherwise.
-- 
2.34.1

