From e30d739f1f21ce09e60f3b30cf3475205e85f249 Mon Sep 17 00:00:00 2001 From: "kuroda.hayato%40jp.fujitsu.com" Date: Mon, 27 Jun 2022 04:20:01 +0000 Subject: [PATCH v4 2/3] Document Logical Replication Group This patch contains documentation for Logical Replication Group(LRG). --- doc/src/sgml/catalogs.sgml | 330 ++++++++++++++++++++++ doc/src/sgml/filelist.sgml | 1 + doc/src/sgml/func.sgml | 121 ++++++++ doc/src/sgml/lrg.sgml | 555 +++++++++++++++++++++++++++++++++++++ doc/src/sgml/postgres.sgml | 1 + 5 files changed, 1008 insertions(+) create mode 100644 doc/src/sgml/lrg.sgml diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index c00c93dd7b..ece5e6819e 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -200,6 +200,26 @@ metadata for large objects + + pg_lrg_info + logical replication group + + + + pg_lrg_nodes + logical replication group + + + + pg_lrg_pub + logical replication group + + + + pg_lrg_sub + logical replication group + + pg_namespace schemas @@ -4970,6 +4990,316 @@ SCRAM-SHA-256$<iteration count>:&l + + <structname>pg_lrg_info</structname> + + + pg_lrg_info + + + The catalog pg_lrg_info stores information about + node groups. A node group is a group of nodes that send and receive their + data changes, and it can be created by the lrg_create + function. See for more information about node groups. + + + + Each database in the same node group has a tuple with the same information + except for oid. + + + + Currently, a node can participate in only one node group. This means that + this system catalog has at most one tuple. + + + + <structname>pg_lrg_info</structname> Columns + + + + + Column Type + + + Description + + + + + + + + oid oid + + + Row identifier + + + + + + groupname name + + + Name of the group + + + + + + pub_type text + + + List of published tables + + + + +
+
+ + + <structname>pg_lrg_nodes</structname> + + + pg_lrg_nodes + + + The catalog pg_lrg_nodes stores information about + nodes in node group that this node is a member of. When lrg_node_attach() + is executed, information about other nodes in a group is copied from the + upstream node, and information about this node is copied to the other nodes. + Therefore, each database in the same node group has tuples with the same + information except for oid and dbid. + See for more information about LRG. + + + + <structname>pg_lrg_nodes</structname> Columns + + + + + Column Type + + + Description + + + + + + + + oid oid + + + Row identifier + + + + + + groupid oid + (references pg_lrg_info.oid) + + + Attached group + + + + + + dbid oid + (references pg_database.oid) + + + Database + + + + + + status int + + + State code: + 0 = initialize, + 1 = a publication was created in this node, + 2 = subscriptions were created in all nodes, + 3 = ready, + 4 = this node is now detaching from a group, + + + + + + nodename name + + + Name of this node + + + + + + nodeid text + + + Identifier of this node + + + + + + localconn text + + + Connection string for this node + + + + + + upstreamconn text + + + Connection string for upstream node + + + + + +
+
+ + + <structname>pg_lrg_pub</structname> + + + pg_lrg_pub + + + The catalog pg_lrg_pub contains the mapping between + node groups and a publication in the database. Since each node is only + allowed to be a member of one group, so this catalog has at most one tuple. + See for more information about LRG. + + + + <structname>pg_lrg_pub</structname> Columns + + + + + Column Type + + + Description + + + + + + + + oid oid + + + Row identifier + + + + + + groupid oid + (references pg_lrg_info.oid) + + + Attached group + + + + + + pubid oid + (references pg_publication.oid) + + + Publication + + + + + + +
+
+ + + + <structname>pg_lrg_sub</structname> + + + pg_lrg_sub + + + The catalog pg_lrg_sub contains the mapping between + node groups and subscriptions in the database. Since each node is only + allowed to be a member of one group, so this catalog has at most N-1 tuples. + See for more information about LRG. + + + + <structname>pg_lrg_sub</structname> Columns + + + + + Column Type + + + Description + + + + + + + + oid oid + + + Row identifier + + + + + + groupid oid + (references pg_lrg_info.oid) + + + Attached group + + + + + + subid oid + (references pg_subscription.oid) + + + Subscription + + + + + + +
+
<structname>pg_namespace</structname> diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 40ef5f7ffc..8be17a652e 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -49,6 +49,7 @@ + diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 7b652460a1..52e4a07604 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -27751,6 +27751,127 @@ postgres=# SELECT * FROM pg_walfile_name_offset((pg_backup_stop()).lsn); + + Logical Replication Groups Management Functions + + + The functions shown in are for + controlling and interacting with node groups. See . + + + + LRG Management Functions + + + + + Function + + + Description + + + + + + + + + + + lrg_create ( group_name name, publication_type text, local_connstring text, node_name name ) + void + + + Creates a node group named group_name and attach + the current node to it, as node_name. Nodes in + the created node group sends and receives change to the specified in + publication_type. If this node is already in a + logical replication group, an error is raised. Note that this function + returns immediately. This function is just a trigger for starting + complex operations by the LRG worker process. If users want to wait + until finishing them, lrg_wait can be used. + + + + + + + + lrg_node_attach ( group_name name, local_connstring text, upstream_connstring text, node_name name ) + void + + + Attach the current node to a node group named group_name + as node_name. The initial data and configuration + information of this group will be shared from the upstream_connstring. + node_name must be unique in the group. + If this node is already in a node group, an error is + raised. Note that this function returns immediately. This function is + just a trigger for starting complex operations by the LRG worker + process. If users want to wait until finishing them, lrg_wait + can be used. + + + + + + + + lrg_node_detach ( group_name name, node_name name , force_detach boolean ) + void + + + Detach the node named node_name from the node + group named group_name. Only nodes in the + specified node group can execute this function, and any nodes in + the node group can be chosen. If this node is not in a + node group, an error is raised. + Note that this function returns immediately. This function is just + a trigger for starting complex operations by the LRG worker process. + If users want to wait until finishing them, lrg_wait + can be used. + By default, LRG worker process tries to drop subscription in nodes, but + it means that it will be failed if specified server goes down. + To avoid that force_detach can be used. When the + parameter is set to true, LRG worker process checks + the health of the node and does special operations. + + + + + + + + lrg_drop ( group_name name ) + void + + + Removes the node group named group_name after + detaching the current node from it. If this node is not in a node group, + or there are still nodes in the group, an error is raised. + + + + + + + + lrg_wait () + void + + + Wait until LRG related operations are done. This function returns + immediately if a node executes this function is not in a node group. + + + + + +
+ +
+ Database Object Management Functions diff --git a/doc/src/sgml/lrg.sgml b/doc/src/sgml/lrg.sgml new file mode 100644 index 0000000000..bb67a82427 --- /dev/null +++ b/doc/src/sgml/lrg.sgml @@ -0,0 +1,555 @@ + + + Logical Replication Groups (LRG) + + + logical replication group + + + + Logical Replication Group (LRG) provides an API that allows a group of nodes + to replicate data to each other. All nodes in the group can execute + Read-Write operatons, and any changes made on one node will "eventually" + replicate to all other member nodes of the group. + + + + Each node in a node group of N member nodes has 1 publication and N-1 + subscriptions to the other N-1 nodes. + + + + LRG is especially useful when working with "data-aware" applications. + It is recommended that users build a node group located across different + geographical locations, where each node inserts or updates different tuples. + + + + Advantages of LRG are: + + + + + Allowing load balancing + + + + + Allowing rolling updates of nodes + + + + + Improving the availability of the system + + + + + + + LRG concepts + + Node group + + + Node group is a group that participant nodes have same tables and + send/receive their data changes. This is done via logical replication. + Each node in a group must be connected to all other nodes. LRG make such a + group by creating publications and subscriptions on nodes. + + + + + + Node + + Node is a database that belongs to a node group and it is identified + by a user-given unique name. By definition, all nodes are not only + publishers, but also subscribers. They must be connected to all other + nodes in a group. If there are N nodes in a node group, each node has + one publication and N-1 subscriptions. + + + + + + + Configuration + + For a node group consisting of N member nodes, LRG will create 1 publication + and N-1 subscriptions on each node. Therefore LRG requires several + configuration options: + + + + + + wal_level must be logical. + + + + + max_replication_slots must be at least N. + + + + + max_wal_senders must be at least max_replication_slots. + + + + + max_logical_replication_workers must be more than N. + + + + + max_worker_processes must be larger that max_logical_replication_workers. + + + + + + + LRG SQL interface + + LRG provides several SQL functions for controlling node group. + See for detailed documentation. + + + + Create a node group + + Node groups are created by lrg_create. + The node that executed this function automatically attaches to the created + group. This function requires a connection string that is used for + connecting to the node from other or connecting to itself. Therefore, it is + not recommended to specify the Unix-domain socket or loopback address as the + connection string unless testing purposes. + + + + When users add a node to a node group, the node name must be specified. + This can be used for managing node information, and it must be specified + when users want to detach a node from a node group. The name must be unique + within a node group. + + + + All tables present in the node group must be set up to allow all changes + to be replicated. This requires either of a primary key, or a replica + identity with unique index and not-null constraint. LRG does not check + them or throw any warnings. Users must define tables properly. Moreover, + users must not change any configurations of the automatically defined + publication. + + + + + + Attach a node to a node group + + In order to attach to a node group, the function lrg_node_attach + can be used. At that time, in addition to specifying connection string for + this node, a connection string for connecting to one of the node in the + group must be specified. The specified node is called as "upstream node". + After executing lrg_node_attach, this node connects + to the upstream node, and gets connection strings for other nodes and + synchronizes the initial data. + + + + Currently, each node can attach to only one node group, and changes to all + tables are replicated to nodes in the group. This is implemented by + defining a publication with target list 'FOR ALL TABLES'. This is an + implementation restriction that might be lifted in a future release. + + + + + + Detach a node from a node group + + The user can detach any attached nodes by executing + lrg_node_detach. This function can be called from any + node that is a member of a node group. This function must be called from a + node that is a member of a group. + + + + + Drop a node group + + Users can remove useless node group via lrg_drop. + This function can only be called from the last remaining node attached to + the node group being dropped; it will throw ERROR if there are other + members in the group. + + + + + + + Locking over nodes + + This feature is under development. + + + + + Architecture + + + LRG implementation uses "LRG launcher" and "LRG worker" processes. The + launcher process is registered when the server is not in the recovery mode, + and it has a responsibility for starting LRG worker processes. The actual + operations for creating or attaching to a node group are done by the LRG + worker process. There can be one LRG worker per database. + + + + More specifically: when SQL functions related with LRG are called by users, + tuples will be inserted to catalogs and backend process will notify to the + launcher. After receiving the notification, the launcher seek + and launches the LRG worker process. This seeking and launching will be + also done when server boots. + + + + + Security + + To execute some APIs, lrg_create, lrg_node_attach, + lrg_node_detach, and lrg_create, + the user must be a superuser. + + + LRG launcher and LRG worker will run with the privileges of a superuser. + + + The user used in the connection string in APIs must be a superuser. This is + because LRG worker uses this string in order to connect to other nodes, + and then creates a subscription in the node. + + + + + + + Examples + + The following example demonstrates constructing a node group. + Assume that there are three nodes, called Node1, Node2, Node3, and port + numbers of them are 5431, 5432, and 5433. All operations will be done by + superuser, named postgres. In the example all nodes are in the same server. + + + + Setup + + Before executing APIs, some configurations must be adjusted. At first + postgresql.conf must be changed as: + +wal_level = logical + + The other parameters have default values that are sufficient for a + basic setup. If you will test same thing in your network configuration, + listen_address should be changed. + + + + pg_hba.conf must be also modified based on your + system. In this examle all postgres nodes are in the same server, we + don't have to care about it. + + + +local all all trust + + + + + + + Create a node group + + + At first, a node group must be created. Here it will be created on Node1. + + +node1=# -- Create a node group 'testgroup', and attach to it as 'testnode1' +node1=# SELECT lrg_create('testgroup', 'FOR ALL TABLES', 'user=postgres port=5431', 'testnode1'); + lrg_create +------------ + +(1 row) + +node1=# -- Wait until operations by LRG worker are done +node1=# SELECT lrg_wait(); + lrg_wait +---------- + +(1 row) + + + + + Information about the node group and nodes can checked via system catalogs: + + +node1=# -- pg_lrg_info, contains information about node group +node1=# SELECT * FROM pg_lrg_info ; + oid | groupname | pub_type +-------+-----------+---------------- + 16389 | testgroup | FOR ALL TABLES +(1 row) + +node1=# -- pg_lrg_nodes, contains information about nodes +node1=# SELECT * FROM pg_lrg_nodes ; + oid | groupid | dbid | status | nodename | nodeid | localconn | upstreamconn +-------+---------+------+--------+-----------+----------------------+-------------------------+-------------- + 16390 | 16389 | 5 | 3 | testnode1 | 71141689070058577515 | user=postgres port=5431 | +(1 row) + +node1=# -- pg_lrg_pub, contains information publication +node1=# SELECT * FROM pg_lrg_pub ; + oid | groupid | pubid +-------+---------+------- + 16392 | 16389 | 16391 +(1 row) + +node1=# -- pg_lrg_pub, contains information subscription +node1=# -- Note that no tuples will be found at the moment - subscriptions will be created when someone attaches to the node group +node1=# SELECT * FROM pg_lrg_sub ; + oid | groupid | subid +-----+---------+------- +(0 rows) + + + + + + Attach a node to a node group + + + Next, the node Node2 can attach to the node group. Note that the upstream + connection string to Node1 must be specified. + + + +node2=# -- Attach to 'testgroup' as 'testnode2' +node2=# SELECT lrg_node_attach('testgroup', 'user=postgres port=5432', 'user=postgres port=5431', 'testnode2'); + lrg_node_attach +----------------- + +(1 row) + +node2=# -- Wait until operations by LRG worker are done +node2=# SELECT lrg_wait(); + lrg_wait +---------- + +(1 row) + + + + The status of all nodes can check via pg_lrg_nodes. Following tuples will be found on both nodes. + + + +node2=# SELECT * FROM pg_lrg_info ; + oid | groupname | pub_type +-------+-----------+---------------- + 16389 | testgroup | FOR ALL TABLES +(1 row) + +node2=# SELECT * FROM pg_lrg_nodes ; + oid | groupid | dbid | status | nodename | nodeid | localconn | upstreamconn +-------+---------+------+--------+-----------+----------------------+-------------------------+------------------------- + 16391 | 16389 | 5 | 3 | testnode1 | 71141695677943317765 | user=postgres port=5431 | + 16390 | 16389 | 5 | 3 | testnode2 | 71141695737527092585 | user=postgres port=5432 | user=postgres port=5431 +(2 rows) + +node2=# SELECT * FROM pg_lrg_pub ; + oid | groupid | pubid +-------+---------+------- + 16393 | 16389 | 16392 +(1 row) + +node2=# -- A tuple will be found because now both of nodes have one subscription +node1=# SELECT * FROM pg_lrg_sub ; + oid | groupid | subid +-------+---------+------- + 16395 | 16389 | 16394 +(1 row) + + + + + Now the LRG system catalogs on Node1 and Node2 have same content, so ether + of them can be specified as an upstream node. Below, Node2 is used as the + upstream node for attaching a new node Node3. + + + +node3=# -- Attach to 'testgroup' as 'testnode3' +node3=# SELECT lrg_node_attach('testgroup', 'user=postgres port=5433', 'user=postgres port=5432', 'testnode3'); + lrg_node_attach +----------------- + +(1 row) + +node3=# -- Wait until operations by LRG worker are done +node3=# SELECT lrg_wait(); + lrg_wait +---------- + +(1 row) + + + + Finally pg_lrg_info and pg_lrg_nodes will be like: + + + +node3=# SELECT * FROM pg_lrg_info ; + oid | groupname | pub_type +-------+-----------+---------------- + 16393 | testgroup | FOR ALL TABLES +(1 row) + +node3=# SELECT * FROM pg_lrg_nodes ; + oid | groupid | dbid | status | nodename | nodeid | localconn | upstreamconn +-------+---------+------+--------+-----------+----------------------+-------------------------+------------------------- + 16391 | 16389 | 5 | 3 | testnode1 | 71141714630795390835 | user=postgres port=5431 | + 16392 | 16389 | 5 | 3 | testnode2 | 71141714693889355845 | user=postgres port=5432 | user=postgres port=5431 + 16390 | 16389 | 5 | 3 | testnode3 | 71141714753722290355 | user=postgres port=5433 | user=postgres port=5432 +(3 rows) + +node3=# SELECT * FROM pg_lrg_pub ; + oid | groupid | pubid +-------+---------+------- + 16394 | 16389 | 16393 +(1 row) + +node3=# -- Two tuple will be found because the # of subscriptions is N-1 +node3=# SELECT * FROM pg_lrg_sub ; + oid | groupid | subid +-------+---------+------- + 16396 | 16389 | 16395 + 16398 | 16389 | 16397 +(2 rows) + + + + + Now each node in the group publishes their changes, and each node + subscribes to every other node in the group. + + + + + + Detach a node from a node group + + Consider a case that users want to shrink its system and remove a node from + a node group. lrg_node_detach can be used for that. + + + +node3=# -- Detach Node3 from 'testgroup'. This can be done from any arbitrary member of the node group. +node3=# SELECT lrg_node_detach('testgroup', 'testnode3', false); + lrg_node_detach +----------------- + +(1 row) + + + + The user-defined data will be not removed even if nodes are detached, but + tuples in LRG system catalogs will be deleted. + + + +node3=# -- On Node3, there are no tuples in pg_lrg_nodes +node3=# select * from pg_lrg_nodes; + oid | nodeid | groupid | dbid | status | nodename | localconn | upstreamconn +-----+--------+---------+------+--------+----------+-----------+-------------- +(0 rows) + + + + By default + + + +node1=# -- Detach Node1 from 'testgroup' as "force" mode - it is useful if the specified node goes down +node1=# SELECT lrg_node_detach('testgroup', 'testnode2', true); + lrg_node_detach +----------------- + +(1 row) + + + + + + Drop a node group + + Suppose that user finishes operations of a system and wants to remove + the node group. In the previous part Node2 and Node3 has been detached, + so the testgroup can be removed via lrg_drop. + + + +node1=# -- Drop 'testgroup'. Node1 will detach from it automatically. +node1=# SELECT lrg_drop('testgroup'); + lrg_drop +---------- + +(1 row) + +node1=# -- there are no tuples in pg_lrg_info +node1=# SELECT * from pg_lrg_info ; + oid | groupname | pub_type +-----+-----------+---------- +(0 rows) + + + + + + + Restrictions + + LRG currently has the following restrictions or missing functionality. + + + + Each node can attach only one node group at a time. + + + + + All tables in the database must be replicated. That is, only + 'FOR ALL TABLES' can be specified for publication_type + in lrg_create. + + + + + All replicated objects must be defined on the all nodes. + + + + + LRG does not provide any mechanism for conflict handling (e.g. PK violations). + Avoiding conflicts is the user responsibility. + + + + + + + + diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml index 0b60e46d69..d7c36c1197 100644 --- a/doc/src/sgml/postgres.sgml +++ b/doc/src/sgml/postgres.sgml @@ -170,6 +170,7 @@ break is not needed in a wider output rendering. &diskusage; &wal; &logical-replication; + &lrg; &jit; ®ress; -- 2.27.0