diff --git a/src/test/subscription/Makefile b/src/test/subscription/Makefile index b402cec..cc246f0 100644 --- a/src/test/subscription/Makefile +++ b/src/test/subscription/Makefile @@ -13,9 +13,9 @@ subdir = src/test/subscription top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -EXTRA_INSTALL = contrib/hstore +EXTRA_INSTALL = contrib/hstore src/test/modules/injection_points -export with_icu +export with_icu enable_injection_points check: $(prove_check) diff --git a/src/test/subscription/meson.build b/src/test/subscription/meson.build index 1c0b6ca..703b0d4 100644 --- a/src/test/subscription/meson.build +++ b/src/test/subscription/meson.build @@ -5,7 +5,10 @@ tests += { 'sd': meson.current_source_dir(), 'bd': meson.current_build_dir(), 'tap': { - 'env': {'with_icu': icu.found() ? 'yes' : 'no'}, + 'env': { + 'with_icu': icu.found() ? 'yes' : 'no', + 'enable_injection_points': get_option('injection_points') ? 'yes' : 'no', + }, 'tests': [ 't/001_rep_changes.pl', 't/002_types.pl', @@ -40,6 +43,7 @@ tests += { 't/031_column_list.pl', 't/032_subscribe_use_index.pl', 't/033_run_as_table_owner.pl', + 't/034_tablesync_durable.pl', 't/100_bugs.pl', ], }, diff --git a/src/test/subscription/t/034_tablesync_durable.pl b/src/test/subscription/t/034_tablesync_durable.pl new file mode 100644 index 0000000..972c69f --- /dev/null +++ b/src/test/subscription/t/034_tablesync_durable.pl @@ -0,0 +1,231 @@ +# Copyright (c) 2026, PostgreSQL Global Development Group +# +# Crash the subscriber after the publisher confirms deletion of a table's +# synchronization slot. The table must recover and continue receiving changes. +# An already synchronized table is a control for the main replication stream. +# Stock PostgreSQL leaves FINISHEDCOPY committed and cannot resume the new table. + +use strict; +use warnings FATAL => 'all'; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(time usleep); + +if (($ENV{enable_injection_points} // '') ne 'yes') +{ + plan skip_all => 'Injection points not supported by this build'; +} + +sub table_contents +{ + my ($node, $table) = @_; + return $node->safe_psql('postgres', + "SELECT coalesce(json_agg(t ORDER BY id)::text, '[]') FROM $table t"); +} + +# Put INSERT, UPDATE, and DELETE on both sides of the subscriber crash. +sub mutate_tables +{ + my ($publisher, $first, $last) = @_; + my $sql = ''; + for my $id ($first .. $last) + { + my $deleted = $id - 100; + $sql .= qq[ +BEGIN; +INSERT INTO healthy VALUES ($id, 'new-$id'); +INSERT INTO stuck VALUES ($id, 'new-$id'); +UPDATE healthy SET v = 'updated-$id' WHERE id = 10; +UPDATE stuck SET v = 'updated-$id' WHERE id = 10; +DELETE FROM healthy WHERE id = $deleted; +DELETE FROM stuck WHERE id = $deleted; +COMMIT; +]; + } + $publisher->safe_psql('postgres', $sql); +} + +# The missing-slot failure retries indefinitely. Recognize its exact error +# after restart so that the recovery assertion fails promptly on stock builds, +# rather than waiting for a generic timeout. Both outcomes use that assertion. +sub await_outcome +{ + my ($subscriber, $source_slot, $log_offset) = @_; + my $missing = + qr/ERROR:\s+could not start WAL streaming:\s+ERROR:\s+replication slot "\Q$source_slot\E" does not exist/; + my $deadline = time() + $PostgreSQL::Test::Utils::timeout_default; + while (time() < $deadline) + { + my $state = $subscriber->safe_psql('postgres', + "SELECT srsubstate FROM pg_subscription_rel WHERE srrelid = 'stuck'::regclass" + ); + my $log = slurp_file($subscriber->logfile, $log_offset); + return (0, $&) if $log =~ $missing; + return (1, undef) if $state eq 'r'; + usleep(100_000); + } + die 'subscriber reached neither READY nor the missing-slot error'; +} + +# Test clusters normally disable fsync. Enable it explicitly to exercise local +# crash recovery, and use synchronous commits for COPY and FINISHEDCOPY. +my $publisher = PostgreSQL::Test::Cluster->new('publisher'); +$publisher->init(allows_streaming => 'logical'); +$publisher->append_conf( + 'postgresql.conf', qq[ +fsync = on +synchronous_commit = on +wal_sender_timeout = 0 +]); +$publisher->start; + +my $subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$subscriber->init; +$subscriber->append_conf( + 'postgresql.conf', qq[ +fsync = on +synchronous_commit = on +wal_retrieve_retry_interval = '100ms' +wal_receiver_timeout = 0 +]); +$subscriber->start; +$subscriber->safe_psql('postgres', 'CREATE EXTENSION injection_points'); +for my $node ($publisher, $subscriber) +{ + is($node->safe_psql('postgres', 'SHOW fsync'), + 'on', $node->name . ' uses fsync'); + $node->safe_psql( + 'postgres', qq[ +CREATE TABLE healthy (id integer PRIMARY KEY, v text); +CREATE TABLE stuck (id integer PRIMARY KEY, v text); +]); +} +$publisher->safe_psql( + 'postgres', qq[ +INSERT INTO healthy SELECT i, 'original-' || i FROM generate_series(1,200) i; +INSERT INTO stuck SELECT * FROM healthy; +CREATE PUBLICATION handoff_pub FOR TABLE healthy; +]); +my $publisher_connstr = $publisher->connstr . ' dbname=postgres'; +$subscriber->safe_psql( + 'postgres', qq[ +CREATE SUBSCRIPTION handoff_sub CONNECTION '$publisher_connstr' +PUBLICATION handoff_pub WITH (synchronous_commit = on); +]); + +# Establish healthy replication before adding the table under test. The usual +# wait_for_subscription_sync helper also accepts SYNCDONE, so require READY. +$subscriber->poll_query_until('postgres', + "SELECT srsubstate = 'r' FROM pg_subscription_rel WHERE srrelid = 'healthy'::regclass" +) or die 'healthy table did not reach READY'; +$publisher->poll_query_until('postgres', + "SELECT count(*) = 0 FROM pg_replication_slots WHERE slot_name LIKE 'pg_%_sync_%'" +) or die 'healthy synchronization slot was not removed'; +my $healthy_filenode = $subscriber->safe_psql('postgres', + "SELECT pg_relation_filenode('healthy')"); +is( $publisher->safe_psql( + 'postgres', + "SELECT count(*) FROM pg_replication_slots WHERE slot_name = 'handoff_sub'" + ), + '1', + 'main publisher slot exists'); + +# The hook is after a successful DROP_REPLICATION_SLOT response. Wait for the +# worker's event to hold this exact crash window without a timing-dependent sleep. +my $pause_point = 'tablesync-after-source-slot-drop'; +$subscriber->safe_psql('postgres', + "SELECT injection_points_attach('$pause_point', 'wait')"); +$publisher->safe_psql('postgres', + 'ALTER PUBLICATION handoff_pub ADD TABLE stuck'); +$subscriber->safe_psql('postgres', + 'ALTER SUBSCRIPTION handoff_sub REFRESH PUBLICATION'); +$subscriber->wait_for_event('logical replication tablesync worker', + $pause_point); +my $source_slot = $subscriber->safe_psql( + 'postgres', qq[ +SELECT 'pg_' || s.oid || '_sync_' || 'stuck'::regclass::oid || '_' || c.system_identifier +FROM pg_subscription s CROSS JOIN pg_control_system() c +WHERE s.subname = 'handoff_sub'; +]); + +# A separate connection sees only committed state: FINISHEDCOPY ('f') on stock, +# SYNCDONE ('s') with the fix. The source synchronization slot is already gone. +my $visible_state = $subscriber->safe_psql('postgres', + "SELECT srsubstate FROM pg_subscription_rel WHERE srrelid = 'stuck'::regclass" +); +like($visible_state, qr/\A[fs]\z/, 'paused table has completed its copy'); +diag("committed table state after source slot deletion: $visible_state"); +is( $publisher->safe_psql( + 'postgres', + "SELECT count(*) FROM pg_replication_slots WHERE slot_name = '$source_slot'" + ), + '0', + 'exact source synchronization slot is absent'); + +# Immediate shutdown prevents an open SYNCDONE transaction from committing. +# Restart clears the injection point, so any cleanup retry can run normally. +mutate_tables($publisher, 201, 205); +my $log_offset = -s $subscriber->logfile; +$subscriber->stop('immediate'); +$subscriber->start; +mutate_tables($publisher, 206, 230); +my ($recovered, $missing_error) = + await_outcome($subscriber, $source_slot, $log_offset); +ok($recovered, 'copied table recovers after subscriber crash'); +diag($missing_error) if defined $missing_error; + +if ($recovered) +{ + # READY alone does not prove that the data caught up. + my $expected = table_contents($publisher, 'stuck'); + $subscriber->poll_query_until('postgres', + "SELECT coalesce(json_agg(t ORDER BY id)::text, '[]') FROM stuck t", + $expected) + or die 'recovered table contents did not catch up'; + is(table_contents($subscriber, 'stuck'), + $expected, 'recovered table has exact contents'); + $subscriber->poll_query_until('postgres', + q{SELECT count(*) = 0 FROM pg_replication_origin WHERE roname ~ '^pg_[0-9]+_[0-9]+$'} + ) or die 'synchronization origin remains after recovery'; +} +else +{ + # These observations distinguish this fault from an unrelated setup error. + is( $subscriber->safe_psql( + 'postgres', + "SELECT srsubstate FROM pg_subscription_rel WHERE srrelid = 'stuck'::regclass" + ), + 'f', + 'missing source slot leaves the table in FINISHEDCOPY'); + isnt( + table_contents($subscriber, 'stuck'), + table_contents($publisher, 'stuck'), + 'missing-slot failure leaves the table behind later source changes'); +} + +# The main stream must keep receiving changes without replacing the healthy +# table. Synchronization cleanup must also preserve the main source slot. +my $expected_healthy = table_contents($publisher, 'healthy'); +$subscriber->poll_query_until('postgres', + "SELECT coalesce(json_agg(t ORDER BY id)::text, '[]') FROM healthy t", + $expected_healthy) + or die 'healthy table contents did not catch up'; +is(table_contents($subscriber, 'healthy'), + $expected_healthy, 'healthy table continues with exact contents'); +is( $subscriber->safe_psql( + 'postgres', "SELECT pg_relation_filenode('healthy')"), + $healthy_filenode, + 'healthy table is not replaced'); +is( $publisher->safe_psql( + 'postgres', + "SELECT count(*) FROM pg_replication_slots WHERE slot_name = 'handoff_sub'" + ), + '1', + 'main source slot remains'); +$publisher->poll_query_until('postgres', + "SELECT count(*) = 0 FROM pg_replication_slots WHERE slot_name LIKE 'pg_%_sync_%'" +) or die 'synchronization slot remains after recovery'; + +done_testing();