From 48cbb4ca2f8a5fd44825554224e0c5f2db5c951b Mon Sep 17 00:00:00 2001 From: Anthony Hsu Date: Sun, 15 Nov 2026 22:59:42 +0000 Subject: [PATCH v4] Set 1s WaitLatch timeout if standby limit has expired in ResolveRecoveryConflictWithBufferPin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Once the standby limit has expired, the startup process signals all backends once and then sleeps until UnpinBuffer() wakes it up. A backend that pins the buffer after it has processed the signal is never told to cancel, so replay can stay stuck for as long as that pin is held. With this change, after standby limit has expired, the startup process will wake up once per second and re-broadcast the buffer pin conflict signal, so that new pinners will receive it and cancel themselves. Reported-by: Anthony Hsu Author: Anthony Hsu Reviewed-by: Dmytro Astapov Reviewed-by: Shihao Zhong Reviewed-by: Andrey Borodin Reviewed-by: João Rito Discussion: https://postgr.es/m/CALQc50gi-Kw9m1r6hytf12473-fCECy=q9JtKS4ANeJFEyCBTw@mail.gmail.com Backpatch-through: 15 --- src/backend/storage/ipc/standby.c | 27 ++++- src/test/recovery/t/031_recovery_conflict.pl | 100 ++++++++++++++++++- 2 files changed, 125 insertions(+), 2 deletions(-) diff --git a/src/backend/storage/ipc/standby.c b/src/backend/storage/ipc/standby.c index 7f011e04990..f0e09b1305b 100644 --- a/src/backend/storage/ipc/standby.c +++ b/src/backend/storage/ipc/standby.c @@ -26,6 +26,7 @@ #include "pgstat.h" #include "replication/slot.h" #include "storage/bufmgr.h" +#include "storage/latch.h" #include "storage/proc.h" #include "storage/procarray.h" #include "storage/sinvaladt.h" @@ -71,6 +72,12 @@ static volatile sig_atomic_t got_standby_deadlock_timeout = false; static volatile sig_atomic_t got_standby_delay_timeout = false; static volatile sig_atomic_t got_standby_lock_timeout = false; +/* + * Frequency with which to resend buffer pin conflicts, in case new pinners + * snuck in after we did our last broadcast. + */ +#define BUFFERPIN_CONFLICT_RESEND_MS 1000 + static void ResolveRecoveryConflictWithVirtualXIDs(VirtualTransactionId *waitlist, RecoveryConflictReason reason, uint32 wait_event_info, @@ -795,6 +802,7 @@ void ResolveRecoveryConflictWithBufferPin(void) { TimestampTz ltime; + bool limit_expired = false; Assert(InHotStandby); @@ -806,6 +814,7 @@ ResolveRecoveryConflictWithBufferPin(void) * We're already behind, so clear a path as quickly as possible. */ SendRecoveryConflictWithBufferPin(RECOVERY_CONFLICT_BUFFERPIN); + limit_expired = true; } else { @@ -841,8 +850,24 @@ ResolveRecoveryConflictWithBufferPin(void) * above can wake us up here. WakeupRecovery() called by walreceiver or * SIGHUP signal handler, etc cannot do that because it uses the different * latch from that ProcWaitForSignal() waits on. + * + * If the limit has already expired, no timeout is armed. A backend that + * pins the buffer after it has processed our signal would then never be + * told to cancel, and UnpinBuffer() only wakes us when the last other pin + * goes away. So wake up periodically, to let the caller recheck the buffer + * and come back here to resend the signal. */ - ProcWaitForSignal(WAIT_EVENT_BUFFER_CLEANUP); + if (limit_expired) + { + (void) WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + BUFFERPIN_CONFLICT_RESEND_MS, + WAIT_EVENT_BUFFER_CLEANUP); + ResetLatch(MyLatch); + CHECK_FOR_INTERRUPTS(); + } + else + ProcWaitForSignal(WAIT_EVENT_BUFFER_CLEANUP); if (got_standby_delay_timeout) SendRecoveryConflictWithBufferPin(RECOVERY_CONFLICT_BUFFERPIN); diff --git a/src/test/recovery/t/031_recovery_conflict.pl b/src/test/recovery/t/031_recovery_conflict.pl index d3f4f591661..e58ba88b99a 100644 --- a/src/test/recovery/t/031_recovery_conflict.pl +++ b/src/test/recovery/t/031_recovery_conflict.pl @@ -9,6 +9,7 @@ use warnings FATAL => 'all'; use PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils; use Test::More; +use Time::HiRes qw(usleep); # Set up nodes @@ -297,7 +298,104 @@ cmp_ok( $node_standby->safe_psql( ); -## RECOVERY CONFLICT 6: Database conflict +## RECOVERY CONFLICT 6: Buffer pin conflict after standby limit expired +$sect = "buffer pin conflict after standby limit expired"; + +# Set up a single-page table with 50 live tuples and 50 dead tuples. +my $table_pin = "bufferpin_test"; +$node_primary->safe_psql( + $test_db, + qq[ + CREATE TABLE $table_pin (id int PRIMARY KEY, pad text) + WITH (autovacuum_enabled = false); + INSERT INTO $table_pin + SELECT g, repeat('x', 20) FROM generate_series(1, 100) g; + DELETE FROM $table_pin WHERE id % 2 = 0; + ]); +$node_primary->wait_for_replay_catchup($node_standby); + +is( $node_standby->safe_psql($test_db, qq[SELECT count(*) FROM $table_pin;]), + 50, "$sect: standby sees 50 live tuples"); + +# Start a reader on $psql_standby that holds a pin on $table_pin and gets +# stuck in ClientWrite because we stop pumping its stdout pipe. +my $clientwrite_pid = $psql_standby->query_safe('SELECT pg_backend_pid();'); +$psql_standby->{stdin} .= qq[ + COPY ( + SELECT t.id, g, repeat('x', 4000) + FROM $table_pin t + CROSS JOIN LATERAL generate_series(1, 10000 + t.id * 0) g + ) TO STDOUT; +]; +$psql_standby->{run}->pump() while $psql_standby->{stdin} ne ''; + +ok( $node_standby->poll_query_until( + $test_db, + qq[SELECT wait_event = 'ClientWrite' FROM pg_stat_activity + WHERE pid = $clientwrite_pid;], + 't'), + "$sect: COPY session is blocked in ClientWrite"); + +$log_location = -s $node_standby->logfile; + +# VACUUM on primary generates a Heap2 prune WAL record requiring a buffer +# cleanup lock on the standby. +$node_primary->safe_psql($test_db, qq[VACUUM $table_pin;]); + +# Wait for the startup process to block on the buffer cleanup lock and wait +# past max_standby_streaming_delay (50ms) so that the startup process has +# already attempted to cancel the ClientWrite session and gone back to waiting +# on the buffer pin. +$log_location = $node_standby->wait_for_log( + qr/recovery still waiting after .* recovery conflict on buffer pin/, + $log_location); +usleep(50_000); + +ok( $node_standby->poll_query_until( + $test_db, + qq[SELECT wait_event = 'BufferCleanup' FROM pg_stat_activity + WHERE backend_type = 'startup';], + 't'), + "$sect: startup process is waiting on BufferCleanup"); + +# Open a second session on standby to acquire another conflicting pin on the +# same page and stay idle in transaction (it was never signaled yet). +my $psql_standby2 = + $node_standby->background_psql($test_db, on_error_stop => 0); +my $standby2_pid = $psql_standby2->query_safe('SELECT pg_backend_pid();'); +$res = $psql_standby2->query_safe( + qq[ + BEGIN; + DECLARE c CURSOR FOR SELECT * FROM $table_pin; + FETCH 5 FROM c; + ]); +like($res, qr/^1\|x{20}$/m, + "$sect: second session with conflicting pin established"); + +$log_location = -s $node_standby->logfile; + +# Kill the ClientWrite client so its backend drops its buffer pin, leaving +# only the second session's pin behind. +$psql_standby->{run}->kill_kill; +ok( $node_standby->poll_query_until( + $test_db, + qq[SELECT count(*) = 0 FROM pg_stat_activity + WHERE pid = $clientwrite_pid;], + 't'), + "$sect: ClientWrite session terminated"); + +# Replay should cancel the second session and catch up. +$node_primary->wait_for_replay_catchup($node_standby); + +check_conflict_log( + "\\[$standby2_pid\\].*User was holding shared buffer pin for too long"); + +# Reset for next test +$psql_standby2->quit; +$psql_standby->reconnect_and_clear(); + + +## RECOVERY CONFLICT 7: Database conflict $sect = "database conflict"; $node_primary->safe_psql('postgres', qq[DROP DATABASE $test_db;]); -- 2.56.0.679.gc527c65306-goog