From 864ef6b3ed5fcbb90af71682a45a5118b3b61845 Mon Sep 17 00:00:00 2001 From: Julien Tachoires Date: Mon, 28 Oct 2024 13:40:28 +0100 Subject: [PATCH 6/6] Add ReorderBuffer ondisk compression TAP tests --- src/test/subscription/Makefile | 2 + src/test/subscription/meson.build | 7 +- .../t/034_reorderbuffer_compression.pl | 107 ++++++++++++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 src/test/subscription/t/034_reorderbuffer_compression.pl diff --git a/src/test/subscription/Makefile b/src/test/subscription/Makefile index ce1ca43009..9341f1493c 100644 --- a/src/test/subscription/Makefile +++ b/src/test/subscription/Makefile @@ -16,6 +16,8 @@ include $(top_builddir)/src/Makefile.global EXTRA_INSTALL = contrib/hstore export with_icu +export with_lz4 +export with_zstd check: $(prove_check) diff --git a/src/test/subscription/meson.build b/src/test/subscription/meson.build index c591cd7d61..772eeb817f 100644 --- a/src/test/subscription/meson.build +++ b/src/test/subscription/meson.build @@ -5,7 +5,11 @@ tests += { 'sd': meson.current_source_dir(), 'bd': meson.current_build_dir(), 'tap': { - 'env': {'with_icu': icu.found() ? 'yes' : 'no'}, + 'env': { + 'with_icu': icu.found() ? 'yes' : 'no', + 'with_lz4': lz4.found() ? 'yes' : 'no', + 'with_zstd': zstd.found() ? 'yes' : 'no', + }, 'tests': [ 't/001_rep_changes.pl', 't/002_types.pl', @@ -40,6 +44,7 @@ tests += { 't/031_column_list.pl', 't/032_subscribe_use_index.pl', 't/033_run_as_table_owner.pl', + 't/034_reorderbuffer_compression.pl', 't/100_bugs.pl', ], }, diff --git a/src/test/subscription/t/034_reorderbuffer_compression.pl b/src/test/subscription/t/034_reorderbuffer_compression.pl new file mode 100644 index 0000000000..57def74bf9 --- /dev/null +++ b/src/test/subscription/t/034_reorderbuffer_compression.pl @@ -0,0 +1,107 @@ + +# Copyright (c) 2024, PostgreSQL Global Development Group + +# Test ReorderBuffer compression +use strict; +use warnings FATAL => 'all'; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +sub test_reorderbuffer_compression +{ + my ($node_publisher, $node_subscriber, $appname, $compression, $compression_rate) = @_; + + # Set subscriber's spill_compression option + $node_subscriber->safe_psql('postgres', + "ALTER SUBSCRIPTION tap_sub SET (spill_compression = $compression)"); + + # Make sure the table is empty + $node_publisher->safe_psql('postgres', 'TRUNCATE test_tab'); + + # Reset replication slot stats + $node_publisher->safe_psql('postgres', + "SELECT pg_stat_reset_replication_slot('tap_sub')"); + + # Insert 1 million rows in the table + $node_publisher->safe_psql('postgres', + "INSERT INTO test_tab SELECT i, 'Message number #'||i::TEXT FROM generate_series(1, 1000000) as i" + ); + + $node_publisher->wait_for_catchup($appname); + + # Check if table content is replicated + my $result = + $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM test_tab"); + is($result, qq(1000000), 'check data was copied to subscriber'); + + # Check if the transaction was spilled on disk + my $res_stats = + $node_publisher->safe_psql('postgres', + "SELECT spill_txns FROM pg_catalog.pg_stat_get_replication_slot('tap_sub');"); + is($res_stats, qq(1), 'check if the transaction was spilled on disk'); + + # Check the compression ratio + my $res_comp_rate = + $node_publisher->safe_psql('postgres', + "SELECT ((1 - spill_write_bytes::FLOAT / spill_bytes::FLOAT) * 100)::INT FROM pg_catalog.pg_stat_get_replication_slot('tap_sub');"); + ok($res_comp_rate >= $compression_rate, "check if the compression rate (spill_compression = '$compression') is greater than or equal to $compression_rate"); +} + +# Create publisher node +my $node_publisher = PostgreSQL::Test::Cluster->new('publisher'); +$node_publisher->init(allows_streaming => 'logical'); +$node_publisher->append_conf('postgresql.conf', + 'logical_decoding_work_mem = 64'); +$node_publisher->start; + +# Create subscriber node +my $node_subscriber = PostgreSQL::Test::Cluster->new('subscriber'); +$node_subscriber->init; +$node_subscriber->start; + +# Setup structure on publisher +$node_publisher->safe_psql('postgres', + "CREATE TABLE test_tab (a int primary key, b text)"); + +# Setup structure on subscriber +$node_subscriber->safe_psql('postgres', + "CREATE TABLE test_tab (a int primary key, b text)"); + +# Setup logical replication +my $publisher_connstr = $node_publisher->connstr . ' dbname=postgres'; +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION tap_pub FOR TABLE test_tab"); + +my $appname = 'tap_sub'; + +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION tap_sub CONNECTION '$publisher_connstr application_name=$appname' PUBLICATION tap_pub WITH (streaming = off)" +); + +# No data compression expected, compression ratio is 0 +test_reorderbuffer_compression($node_publisher, $node_subscriber, $appname, + 'off', 0); +# Compression ratio greater than or equal to 30% for pglz +test_reorderbuffer_compression($node_publisher, $node_subscriber, $appname, + 'pglz', 30); +SKIP: +{ + skip "LZ4 not supported by this build", 2 if ($ENV{with_lz4} ne 'yes'); + # Compression ratio greater than or equal to 70% for lz4 + test_reorderbuffer_compression($node_publisher, $node_subscriber, $appname, + 'lz4', 70); +} +SKIP: +{ + skip "ZSTD not supported by this build", 2 if ($ENV{with_zstd} ne 'yes'); + # Compression ratio greater than or equal to 80% for zstd + test_reorderbuffer_compression($node_publisher, $node_subscriber, $appname, + 'zstd', 80); +} + +$node_subscriber->stop; +$node_publisher->stop; + +done_testing(); -- 2.43.0