From b9e897487cf5a03b4bd0c53e4050f07fea96181a Mon Sep 17 00:00:00 2001 From: Bharath Rupireddy Date: Thu, 14 Oct 2021 10:53:40 +0000 Subject: [PATCH v1] add retry mechanism with a GUC before failing the standby with recovery ended before configured recovery fatal error --- src/backend/access/transam/xlog.c | 73 +++++++++++++++++++++++++++++++ src/backend/utils/misc/guc.c | 12 +++++ src/include/access/xlog.h | 1 + 3 files changed, 86 insertions(+) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index ffc9fa5bf1..9b62a68870 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -108,6 +108,7 @@ int CommitDelay = 0; /* precommit delay in microseconds */ int CommitSiblings = 5; /* # concurrent xacts needed to sleep */ int wal_retrieve_retry_interval = 5000; int max_slot_wal_keep_size_mb = -1; +int recovery_target_retry_timeout = 30; #ifdef WAL_DEBUG bool XLOG_DEBUG = false; @@ -968,6 +969,7 @@ static void WALInsertLockAcquire(void); static void WALInsertLockAcquireExclusive(void); static void WALInsertLockRelease(void); static void WALInsertLockUpdateInsertingAt(XLogRecPtr insertingAt); +static bool WaitForRecoveryTarget(void); /* * Insert an XLOG record represented by an already-constructed chain of data @@ -7385,6 +7387,34 @@ StartupXLOG(void) /* Else, try to fetch the next WAL record */ record = ReadRecord(xlogreader, LOG, false); + + if (ArchiveRecoveryRequested && + recoveryTarget != RECOVERY_TARGET_UNSET && + record == NULL && !reachedRecoveryTarget) + { + bool retry_timeout; + + while (record == NULL) + { + if (recovery_target_retry_timeout > 0) + { + retry_timeout = WaitForRecoveryTarget(); + + /* try to fetch the next WAL record */ + record = ReadRecord(xlogreader, LOG, false); + + /* yay! we've got a record */ + if (record) + break; + + /* we've waited enough, no luck, so let's exit now. */ + if (retry_timeout) + break; + } + else + break; + } + } } while (record != NULL); /* @@ -7957,6 +7987,49 @@ StartupXLOG(void) RequestCheckpoint(CHECKPOINT_FORCE); } +static bool +WaitForRecoveryTarget(void) +{ + static long total_wait_time_msec = 0; + long wait_time_msec = 1000; + + /* calculate total wait time in first call */ + if (total_wait_time_msec == 0) + total_wait_time_msec = recovery_target_retry_timeout * 1000; + + /* quick exit without arming the latch if it's already past time */ + if (total_wait_time_msec <= 0) + { + elog(LOG, "timeout occurred while waiting for recovery target, waited %d seconds", + recovery_target_retry_timeout); + return true; + } + + ResetLatch(&XLogCtl->recoveryWakeupLatch); + + /* might change the trigger file's location */ + HandleStartupProcInterrupts(); + + elog(LOG, "waiting for %ld milliseconds before recovery target retry", wait_time_msec); + + (void) WaitLatch(&XLogCtl->recoveryWakeupLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + wait_time_msec, + WAIT_EVENT_RECOVERY_APPLY_DELAY); + + total_wait_time_msec -= wait_time_msec; + + /* exit if waited enough */ + if (total_wait_time_msec <= 0) + { + elog(LOG, "timeout occurred while waiting for recovery target, waited %d seconds", + recovery_target_retry_timeout); + return true; + } + + return false; +} + /* * Checks if recovery has reached a consistent state. When consistency is * reached and we have a valid starting standby snapshot, tell postmaster diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index ce58018635..f67f454ed2 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -3110,6 +3110,18 @@ static struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"recovery_target_retry_timeout", PGC_SIGHUP, REPLICATION_STANDBY, + gettext_noop("Sets the time to wait before retrying to reach recovery target " + "after a failed attempt."), + NULL, + GUC_UNIT_S + }, + &recovery_target_retry_timeout, + 30, 10, 86400, + NULL, NULL, NULL + }, + { {"wal_segment_size", PGC_INTERNAL, PRESET_OPTIONS, gettext_noop("Shows the size of write ahead log segments."), diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 1570860f95..96db8f7f48 100644 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -131,6 +131,7 @@ extern int recovery_min_apply_delay; extern char *PrimaryConnInfo; extern char *PrimarySlotName; extern bool wal_receiver_create_temp_slot; +extern int recovery_target_retry_timeout; /* indirectly set via GUC system */ extern TransactionId recoveryTargetXid; -- 2.25.1