From 51bb2f8a3be937f33f37676997044758936f116c Mon Sep 17 00:00:00 2001 From: David Christensen Date: Fri, 9 Oct 2020 13:36:50 -0500 Subject: [PATCH] Add `truncate` option to subscription commands When adding new tables to a subscription (either via `CREATE SUBSCRIPTION` or `REFRESH PUBLICATION`), tables on the target which are being newly subscribed will be truncated before the data copy step. This saves explicit coordination of a manual `TRUNCATE` on the target tables and allows the results of the initial data sync to be the same as on the publisher at the time of sync. To preserve compatibility with existing behavior, the default value for this parameter is `false`. --- doc/src/sgml/ref/alter_subscription.sgml | 10 ++++ doc/src/sgml/ref/create_subscription.sgml | 10 ++++ src/backend/commands/subscriptioncmds.c | 60 +++++++++++++++++++-- src/backend/replication/logical/tablesync.c | 34 ++++++++++++ src/include/catalog/pg_subscription_rel.h | 1 + src/test/regress/expected/subscription.out | 2 + src/test/regress/sql/subscription.sql | 1 + src/test/subscription/t/010_truncate.pl | 45 +++++++++++++++- 8 files changed, 157 insertions(+), 6 deletions(-) diff --git a/doc/src/sgml/ref/alter_subscription.sgml b/doc/src/sgml/ref/alter_subscription.sgml index db5e59f707..db5c62ba09 100644 --- a/doc/src/sgml/ref/alter_subscription.sgml +++ b/doc/src/sgml/ref/alter_subscription.sgml @@ -133,6 +133,16 @@ ALTER SUBSCRIPTION name RENAME TO < + + truncate (boolean) + + + Specifies whether newly subscribed tables should be truncated before + copying the initial data. The default is false. + (Previously subscribed tables are not truncated.) + + + diff --git a/doc/src/sgml/ref/create_subscription.sgml b/doc/src/sgml/ref/create_subscription.sgml index e812beee37..5a849cdd1b 100644 --- a/doc/src/sgml/ref/create_subscription.sgml +++ b/doc/src/sgml/ref/create_subscription.sgml @@ -107,6 +107,16 @@ CREATE SUBSCRIPTION subscription_name + + truncate (boolean) + + + Specifies whether newly subscribed tables should be truncated before + copying the initial data. The default is false. + + + + create_slot (boolean) diff --git a/src/backend/commands/subscriptioncmds.c b/src/backend/commands/subscriptioncmds.c index 1696454c0b..39ede63fde 100644 --- a/src/backend/commands/subscriptioncmds.c +++ b/src/backend/commands/subscriptioncmds.c @@ -61,6 +61,7 @@ parse_subscription_options(List *options, bool *create_slot, bool *slot_name_given, char **slot_name, bool *copy_data, + bool *truncate, char **synchronous_commit, bool *refresh, bool *binary_given, bool *binary, @@ -70,6 +71,7 @@ parse_subscription_options(List *options, bool connect_given = false; bool create_slot_given = false; bool copy_data_given = false; + bool truncate_given = false; bool refresh_given = false; /* If connect is specified, the others also need to be. */ @@ -91,6 +93,8 @@ parse_subscription_options(List *options, } if (copy_data) *copy_data = true; + if (truncate) + *truncate = false; if (synchronous_commit) *synchronous_commit = NULL; if (refresh) @@ -165,6 +169,16 @@ parse_subscription_options(List *options, copy_data_given = true; *copy_data = defGetBoolean(defel); } + else if (strcmp(defel->defname, "truncate") == 0 && truncate) + { + if (truncate_given) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + + truncate_given = true; + *truncate = defGetBoolean(defel); + } else if (strcmp(defel->defname, "synchronous_commit") == 0 && synchronous_commit) { @@ -242,6 +256,12 @@ parse_subscription_options(List *options, errmsg("%s and %s are mutually exclusive options", "connect = false", "copy_data = true"))); + if (truncate && truncate_given && *truncate) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("%s and %s are mutually exclusive options", + "connect = false", "truncate = true"))); + /* Change the defaults of other options. */ *enabled = false; *create_slot = false; @@ -280,6 +300,18 @@ parse_subscription_options(List *options, errmsg("subscription with %s must also set %s", "slot_name = NONE", "create_slot = false"))); } + + /* + * Truncate is only valid when copy_data is true. + */ + + if (truncate && *truncate && !*copy_data) + { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("%s can only be specified when %s", + "truncate = true", "copy_data = true"))); + } } /* @@ -353,6 +385,7 @@ CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) bool enabled_given; bool enabled; bool copy_data; + bool truncate; bool streaming; bool streaming_given; char *synchronous_commit; @@ -376,6 +409,7 @@ CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) &create_slot, &slotname_given, &slotname, ©_data, + &truncate, &synchronous_commit, NULL, /* no "refresh" */ &binary_given, &binary, @@ -494,7 +528,10 @@ CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) * Set sync state based on if we were asked to do data copy or * not. */ - table_state = copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY; + if (copy_data) + table_state = truncate ? SUBREL_STATE_INIT_TRUNC : SUBREL_STATE_INIT; + else + table_state = SUBREL_STATE_READY; /* * Get the table list from publisher and build local table status @@ -557,7 +594,7 @@ CreateSubscription(CreateSubscriptionStmt *stmt, bool isTopLevel) } static void -AlterSubscription_refresh(Subscription *sub, bool copy_data) +AlterSubscription_refresh(Subscription *sub, bool copy_data, bool truncate) { char *err; List *pubrel_names; @@ -626,8 +663,15 @@ AlterSubscription_refresh(Subscription *sub, bool copy_data) if (!bsearch(&relid, subrel_local_oids, list_length(subrel_states), sizeof(Oid), oid_cmp)) { + char subrel_state; + + if (copy_data) + subrel_state = (truncate ? SUBREL_STATE_INIT_TRUNC : SUBREL_STATE_INIT); + else + subrel_state = SUBREL_STATE_READY; + AddSubscriptionRelState(sub->oid, relid, - copy_data ? SUBREL_STATE_INIT : SUBREL_STATE_READY, + subrel_state, InvalidXLogRecPtr); ereport(DEBUG1, (errmsg("table \"%s.%s\" added to subscription \"%s\"", @@ -727,6 +771,7 @@ AlterSubscription(AlterSubscriptionStmt *stmt) NULL, /* no "create_slot" */ &slotname_given, &slotname, NULL, /* no "copy_data" */ + NULL, /* no "truncate" */ &synchronous_commit, NULL, /* no "refresh" */ &binary_given, &binary, @@ -784,6 +829,7 @@ AlterSubscription(AlterSubscriptionStmt *stmt) NULL, /* no "create_slot" */ NULL, NULL, /* no "slot_name" */ NULL, /* no "copy_data" */ + NULL, /* no "truncate" */ NULL, /* no "synchronous_commit" */ NULL, /* no "refresh" */ NULL, NULL, /* no "binary" */ @@ -821,6 +867,7 @@ AlterSubscription(AlterSubscriptionStmt *stmt) case ALTER_SUBSCRIPTION_PUBLICATION: { bool copy_data; + bool truncate; bool refresh; parse_subscription_options(stmt->options, @@ -829,6 +876,7 @@ AlterSubscription(AlterSubscriptionStmt *stmt) NULL, /* no "create_slot" */ NULL, NULL, /* no "slot_name" */ ©_data, + &truncate, NULL, /* no "synchronous_commit" */ &refresh, NULL, NULL, /* no "binary" */ @@ -851,7 +899,7 @@ AlterSubscription(AlterSubscriptionStmt *stmt) /* Make sure refresh sees the new list of publications. */ sub->publications = stmt->publication; - AlterSubscription_refresh(sub, copy_data); + AlterSubscription_refresh(sub, copy_data, truncate); } break; @@ -860,6 +908,7 @@ AlterSubscription(AlterSubscriptionStmt *stmt) case ALTER_SUBSCRIPTION_REFRESH: { bool copy_data; + bool truncate; if (!sub->enabled) ereport(ERROR, @@ -872,12 +921,13 @@ AlterSubscription(AlterSubscriptionStmt *stmt) NULL, /* no "create_slot" */ NULL, NULL, /* no "slot_name" */ ©_data, + &truncate, NULL, /* no "synchronous_commit" */ NULL, /* no "refresh" */ NULL, NULL, /* no "binary" */ NULL, NULL); /* no "streaming" */ - AlterSubscription_refresh(sub, copy_data); + AlterSubscription_refresh(sub, copy_data, truncate); break; } diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index c27d970589..f2afcb7882 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -90,6 +90,7 @@ #include "catalog/pg_subscription_rel.h" #include "catalog/pg_type.h" #include "commands/copy.h" +#include "commands/tablecmds.h" #include "miscadmin.h" #include "parser/parse_relation.h" #include "pgstat.h" @@ -858,6 +859,39 @@ LogicalRepSyncTableStart(XLogRecPtr *origin_startpos) switch (MyLogicalRepWorker->relstate) { + case SUBREL_STATE_INIT_TRUNC: + { + Relation rel; + List *relids = NIL; + List *rels = NIL; + + SpinLockAcquire(&MyLogicalRepWorker->relmutex); + MyLogicalRepWorker->relstate = SUBREL_STATE_INIT; + MyLogicalRepWorker->relstate_lsn = InvalidXLogRecPtr; + SpinLockRelease(&MyLogicalRepWorker->relmutex); + + /* Simple attempt to truncate the relation itself */ + StartTransactionCommand(); + rel = table_open(MyLogicalRepWorker->relid, RowExclusiveLock); + + rels = lappend(rels, rel); + relids = lappend_oid(relids, MyLogicalRepWorker->relid); + + ExecuteTruncateGuts(rels, relids, NIL, DROP_RESTRICT, false); + CommitTransactionCommand(); + + table_close(rel, NoLock); + + /* Update the state and make it visible to others. */ + StartTransactionCommand(); + UpdateSubscriptionRelState(MyLogicalRepWorker->subid, + MyLogicalRepWorker->relid, + MyLogicalRepWorker->relstate, + MyLogicalRepWorker->relstate_lsn); + CommitTransactionCommand(); + pgstat_report_stat(false); + } + /* Fallthrough for data sync */ case SUBREL_STATE_INIT: case SUBREL_STATE_DATASYNC: { diff --git a/src/include/catalog/pg_subscription_rel.h b/src/include/catalog/pg_subscription_rel.h index f384f4e7fa..d9f4d5ad42 100644 --- a/src/include/catalog/pg_subscription_rel.h +++ b/src/include/catalog/pg_subscription_rel.h @@ -56,6 +56,7 @@ typedef FormData_pg_subscription_rel *Form_pg_subscription_rel; * ---------------- */ #define SUBREL_STATE_INIT 'i' /* initializing (sublsn NULL) */ +#define SUBREL_STATE_INIT_TRUNC 't' /* initializing, but truncate target (sublsn NULL) */ #define SUBREL_STATE_DATASYNC 'd' /* data is being synchronized (sublsn * NULL) */ #define SUBREL_STATE_SYNCDONE 's' /* synchronization finished in front of diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out index 2fa9bce66a..461b083cfb 100644 --- a/src/test/regress/expected/subscription.out +++ b/src/test/regress/expected/subscription.out @@ -48,6 +48,8 @@ SET SESSION AUTHORIZATION 'regress_subscription_user'; -- fail - invalid option combinations CREATE SUBSCRIPTION regress_testsub2 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, copy_data = true); ERROR: connect = false and copy_data = true are mutually exclusive options +CREATE SUBSCRIPTION regress_testsub2 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (copy_data = false, truncate = true); +ERROR: truncate = true can only be specified when copy_data = true CREATE SUBSCRIPTION regress_testsub2 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, enabled = true); ERROR: connect = false and enabled = true are mutually exclusive options CREATE SUBSCRIPTION regress_testsub2 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, create_slot = true); diff --git a/src/test/regress/sql/subscription.sql b/src/test/regress/sql/subscription.sql index 14fa0b247e..cd49b95c1a 100644 --- a/src/test/regress/sql/subscription.sql +++ b/src/test/regress/sql/subscription.sql @@ -40,6 +40,7 @@ SET SESSION AUTHORIZATION 'regress_subscription_user'; -- fail - invalid option combinations CREATE SUBSCRIPTION regress_testsub2 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, copy_data = true); +CREATE SUBSCRIPTION regress_testsub2 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (copy_data = false, truncate = true); CREATE SUBSCRIPTION regress_testsub2 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, enabled = true); CREATE SUBSCRIPTION regress_testsub2 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (connect = false, create_slot = true); CREATE SUBSCRIPTION regress_testsub2 CONNECTION 'dbname=regress_doesnotexist' PUBLICATION testpub WITH (slot_name = NONE, enabled = true); diff --git a/src/test/subscription/t/010_truncate.pl b/src/test/subscription/t/010_truncate.pl index be2c0bdc35..2a60cf0a94 100644 --- a/src/test/subscription/t/010_truncate.pl +++ b/src/test/subscription/t/010_truncate.pl @@ -3,7 +3,7 @@ use strict; use warnings; use PostgresNode; use TestLib; -use Test::More tests => 9; +use Test::More tests => 11; # setup @@ -41,6 +41,18 @@ $node_publisher->safe_psql('postgres', $node_subscriber->safe_psql('postgres', "CREATE TABLE tab4 (x int PRIMARY KEY, y int REFERENCES tab3)"); +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab5 (a int PRIMARY KEY)"); + +$node_subscriber->safe_psql('postgres', + "CREATE TABLE tab5 (a int PRIMARY KEY)"); + +$node_publisher->safe_psql('postgres', + "CREATE TABLE tab6 (a int PRIMARY KEY)"); + +$node_subscriber->safe_psql('postgres', + "CREATE TABLE tab6 (a int PRIMARY KEY)"); + $node_subscriber->safe_psql('postgres', "CREATE SEQUENCE seq1 OWNED BY tab1.a"); $node_subscriber->safe_psql('postgres', "ALTER SEQUENCE seq1 START 101"); @@ -158,3 +170,34 @@ is($result, qq(0||), 'truncate of multiple tables some not published'); $result = $node_subscriber->safe_psql('postgres', "SELECT count(*), min(a), max(a) FROM tab2"); is($result, qq(3|1|3), 'truncate of multiple tables some not published'); + +# test CREATE SUBSCRIPTION without TRUNCATE option +$node_subscriber->safe_psql('postgres', "INSERT INTO tab5 VALUES (1),(2),(3)"); +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION pub2 FOR TABLE tab5"); +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION sub2 CONNECTION '$publisher_connstr' PUBLICATION pub2" +); +$node_publisher->safe_psql('postgres', "INSERT INTO tab5 VALUES (4)"); +$node_publisher->wait_for_catchup('sub2'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM tab5"); +is($result, qq(4), 'Not specifying TRUNCATE leaves existing table alone'); + +# same, but with TRUNCATE +$node_subscriber->safe_psql('postgres', "DROP SUBSCRIPTION sub2"); +$node_publisher->safe_psql('postgres', "DROP PUBLICATION pub2"); + +$node_subscriber->safe_psql('postgres', "INSERT INTO tab6 VALUES (1),(2),(3)"); +$node_publisher->safe_psql('postgres', + "CREATE PUBLICATION pub2 FOR TABLE tab6"); +$node_subscriber->safe_psql('postgres', + "CREATE SUBSCRIPTION sub2 CONNECTION '$publisher_connstr' PUBLICATION pub2 WITH (truncate = true)" +); +$node_publisher->safe_psql('postgres', "INSERT INTO tab6 VALUES (4)"); +$node_publisher->wait_for_catchup('sub2'); + +$result = $node_subscriber->safe_psql('postgres', + "SELECT count(*) FROM tab6"); +is($result, qq(1), 'with TRUNCATE data is reset'); -- 2.24.3 (Apple Git-128)