From 46e3496b4477cb133674ba8d0c349871f6127baa Mon Sep 17 00:00:00 2001
From: Andres Freund <andres@anarazel.de>
Date: Mon, 27 Jan 2014 17:13:49 +0100
Subject: [PATCH 4/5] wal_decoding: Documentation for replication slots and
 changeset extraction

Andres, Craig
---
 doc/src/sgml/catalogs.sgml            |  27 +-
 doc/src/sgml/changesetextraction.sgml | 527 ++++++++++++++++++++++++++++++++++
 doc/src/sgml/filelist.sgml            |   2 +
 doc/src/sgml/func.sgml                |  94 +++++-
 doc/src/sgml/postgres.sgml            |   1 +
 doc/src/sgml/protocol.sgml            |  95 ++++--
 doc/src/sgml/ref/allfiles.sgml        |   1 +
 doc/src/sgml/ref/create_table.sgml    |   4 +-
 doc/src/sgml/ref/pg_recvlogical.sgml  | 295 +++++++++++++++++++
 doc/src/sgml/reference.sgml           |   1 +
 10 files changed, 1026 insertions(+), 21 deletions(-)
 create mode 100644 doc/src/sgml/changesetextraction.sgml
 create mode 100644 doc/src/sgml/ref/pg_recvlogical.sgml

diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml
index dca24fc..5e680cf 100644
--- a/doc/src/sgml/catalogs.sgml
+++ b/doc/src/sgml/catalogs.sgml
@@ -5177,7 +5177,7 @@
 
   <para>
    For more on replication slots,
-   see <xref linkend="streaming-replication-slots">.
+   see <xref linkend="streaming-replication-slots"> and <xref linkend="changesetextraction">.
   </para>
 
   <table>
@@ -5210,6 +5210,13 @@
      </row>
 
      <row>
+      <entry><structfield>plugin</structfield></entry>
+      <entry><type>text</type></entry>
+      <entry></entry>
+      <entry>The basename of the shared object containing the output plugin this logical slot is using, or null for physical slots.</entry>
+     </row>
+
+     <row>
       <entry><structfield>datoid</structfield></entry>
       <entry><type>oid</type></entry>
       <entry><literal><link linkend="catalog-pg-database"><structname>pg_database</structname></link>.oid</literal></entry>
@@ -5243,6 +5250,24 @@
      </row>
 
      <row>
+      <entry><structfield>xmin</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>The oldest transaction that this slot needs the database to
+      retain.  <literal>VACUUM</literal> cannot remove catalog tuples deleted
+      by any later transaction.
+      </entry>
+     </row>
+
+     <row>
+      <entry><structfield>catalog_xmin</structfield></entry>
+      <entry><type>xid</type></entry>
+      <entry></entry>
+      <entry>The <literal>xmin</literal>, or oldest transaction ID, that this
+      slot forces to be retained in the system catalogs. </entry>
+     </row>
+
+     <row>
       <entry><structfield>restart_lsn</structfield></entry>
       <entry><type>text</type></entry>
       <entry></entry>
diff --git a/doc/src/sgml/changesetextraction.sgml b/doc/src/sgml/changesetextraction.sgml
new file mode 100644
index 0000000..38ac4ea
--- /dev/null
+++ b/doc/src/sgml/changesetextraction.sgml
@@ -0,0 +1,527 @@
+<!-- doc/src/sgml/changesetextraction.sgml -->
+ <chapter id="changesetextraction">
+  <title>Changeset Extraction</title>
+  <indexterm zone="changesetextraction">
+   <primary>Changeset Extraction</primary>
+  </indexterm>
+  <para>
+   PostgreSQL provides infrastructure to stream the modifications performed
+   via SQL to external consumers which can be used to implement replication
+   solutions, perform auditing and similar tasks.
+  </para>
+
+  <para>
+   <indexterm><primary>Changeset Extraction Slot</primary></indexterm>
+   A changeset extraction slot ("logical replication slot") is a persistent
+   server-side record of the replay progress of a stream of changes. A stream of
+   changes is read from the slot to a receiving client program.
+  </para>
+  <para>
+   The format in which those changes are streamed is determined by the output
+   plugin used. While an example plugin is provided, additional plugins can be
+   written to extend the choice of available formats without modifying any
+   core code.
+  </para>
+  <para>
+   Changes can be consumed either using one streaming replication protocol
+   (see <xref linkend="protocol-replication"> and
+   <xref linkend="changesetextraction-walsender">), or by calling functions
+   via SQL (see <xref linkend="changesetextraction-sql">). It is also possible
+   to write additional methods of consuming the output of a replication slot
+   without modifying core code
+   (see <xref linkend="changesetextraction-writer">).
+  </para>
+
+  <sect1 id="changesetextraction-example">
+   <title>Changeset Extraction Example</title>
+   <para>
+    The following example shows usage of the SQL interface.
+   </para>
+   <para>
+    Before you can use changeset extraction you must edit
+    <literal>postgresql.conf</literal> and ensure the following parameters are
+    set to at least:
+    <programlisting>
+wal_level = logical
+max_replication_slots = 1
+    </programlisting>
+    and restart PostgreSQL. Then connect to the target database (in the example
+    below, <literal>postgres</literal>) as a superuser.
+   </para>
+   <programlisting>
+postgres=# -- max_replication_slots must be nonzero and wal_level must be logical
+postgres=# SHOW max_replication_slots;
+ max_replication_slots
+-----------------------
+ 1
+(1 row)
+
+postgres=# SHOW wal_level;
+ wal_level
+-----------
+ logical
+(1 row)
+
+postgres=# -- The decoding plugin you want must be installed. If no rows are returned,
+postgres=# -- then cd contrib/test_decoding && make install then re-check.
+postgres=#
+postgres=# select * from pg_available_extensions WHERE name = 'test_decoding';
+     name      | default_version | installed_version |                              comment
+---------------+-----------------+-------------------+-------------------------------------------------------------------
+ test_decoding | 1.0             | 1.0               | test output plugin for changeset extraction / logical replication
+(1 row)
+
+postgres=# SELECT * FROM pg_replication_slots;
+ slot_name | plugin | slottype | datoid | database | active | catalog_xmin | data_xmin | restart_decoding_lsn
+-----------+--------+----------+--------+----------+--------+--------------+-----------+----------------------
+(0 rows)
+
+postgres=# -- Create a slot named 'regression_slot' using the output plugin 'test_decoding'
+postgres=# SELECT * FROM pg_create_decoding_replication_slot('regression_slot', 'test_decoding');
+    slotname     | xlog_position
+-----------------+---------------
+ regression_slot | 0/16B1970
+
+postgres=# SELECT * FROM pg_replication_slots;
+    slot_name    |    plugin     | slottype | datoid | database | active | catalog_xmin | data_xmin | restart_decoding_lsn
+-----------------+---------------+----------+--------+----------+--------+--------------+-----------+----------------------
+ regression_slot | test_decoding | logical  |  12054 | postgres | f      |          690 |         0 | 0/16B1938
+(1 row)
+
+postgres=# -- There are no changes to see yet
+postgres=# SELECT * FROM pg_decoding_slot_get_changes('regression_slot', 'now', 'include-xids', '0');
+ location | xid | data
+----------+-----+------
+(0 rows)
+
+postgres=# CREATE TABLE data(id serial primary key, data text);
+CREATE TABLE
+
+postgres=# -- DDL isn't replicated, so all you'll see is the transaction
+postgres=# SELECT * FROM pg_decoding_slot_get_changes('regression_slot', 'now', 'include-xids', '0');
+ location  | xid |  data
+-----------+-----+--------
+ 0/16E1558 | 716 | BEGIN
+ 0/16EBAC8 | 716 | COMMIT
+(2 rows)
+
+postgres=# -- Once changes are read once, they're consumed and not emitted
+postgres=# -- in a subsequent call:
+postgres=# SELECT * FROM pg_decoding_slot_get_changes('regression_slot', 'now', 'include-xids', '0');
+ location | xid | data
+----------+-----+------
+(0 rows)
+
+postgres=# BEGIN;
+postgres=# INSERT INTO data(data) VALUES('1');
+postgres=# INSERT INTO data(data) VALUES('1');
+postgres=# COMMIT;
+
+postgres=# SELECT * FROM pg_decoding_slot_get_changes('regression_slot', 'now', 'include-xids', '0');
+ location  | xid |                     data
+-----------+-----+-----------------------------------------------
+ 0/16EBD10 | 718 | BEGIN
+ 0/16EBE60 | 718 | table "data": INSERT: id[int4]:2 data[text]:1
+ 0/16EBE60 | 718 | table "data": INSERT: id[int4]:3 data[text]:1
+ 0/16EBE60 | 718 | COMMIT
+(4 rows)
+
+postgres=# INSERT INTO data(data) VALUES('1');
+
+postgres=# -- You can also peek ahead in the change stream without consuming changes
+postgres=# SELECT * FROM pg_decoding_slot_peek_changes('regression_slot', 'now', 'include-xids', '0');
+ location  | xid |                     data
+-----------+-----+-----------------------------------------------
+ 0/16EBE98 | 719 | BEGIN
+ 0/16EBF58 | 719 | table "data": INSERT: id[int4]:4 data[text]:1
+ 0/16EBF58 | 719 | COMMIT
+(3 rows)
+
+postgres=# SELECT * FROM pg_decoding_slot_peek_changes('regression_slot', 'now', 'include-xids', '0');
+ location  | xid |                     data
+-----------+-----+-----------------------------------------------
+ 0/16EBE98 | 719 | BEGIN
+ 0/16EBF58 | 719 | table "data": INSERT: id[int4]:4 data[text]:1
+ 0/16EBF58 | 719 | COMMIT
+
+postgres=# -- Remember to destroy a slot you no longer need to stop it consuming
+postgres=# -- server resources:
+postgres=# SELECT pg_drop_replication_slot('regression_slot');
+ pg_drop_replication_slot
+-----------------------
+
+(1 row)
+    </programlisting>
+   <para>
+    The following example shows usage of the walsender interface using
+    the <link linkend="app-pgrecvlogical"><command>pg_recvlogical</command></link>
+    shell command. It requires the replication configurations to be allowed
+    (see <xref linkend="streaming-replication-authentication">)
+    and <varname>max_wal_senders</varname> to be set sufficiently high for
+    another connection.
+   </para>
+   <programlisting>
+# pg_recvlogical --slot test --init -d testdb
+# pg_recvlogical --slot test -f - --start -d testdb
+CTRL-Z
+# psql -c "INSERT INTO data(data) VALUES('1');"
+# fg
+BEGIN 721
+table "data": INSERT: id[int4]:5 data[text]:1
+COMMIT 721
+   </programlisting>
+  </sect1>
+  <sect1 id="changesetextraction-explanation">
+   <title>Changeset Extraction Concepts</title>
+   <sect2>
+    <indexterm>
+     <primary>Changeset Extraction</primary>
+    </indexterm>
+    <indexterm>
+     <primary>Changeset Generation</primary>
+    </indexterm>
+    <title>Changeset Extraction</title>
+    <para>
+     Changeset Extraction is the the process of extracting all persistent
+     changes to a database's tables into a coherent, easy to understand format
+     which is easy to interpret independent of the way the changes were made.
+    </para>
+    <para>
+     In <productname>PostgreSQL</productname> changeset extraction is
+     implemented by decoding the <link linkend="wal">WAL's</link> contents,
+     which describe changes on a storage level, into an easier to interpret
+     format. That process is called logical decoding.
+    </para>
+   </sect2>
+   <sect2>
+    <indexterm>
+     <primary>Logical Replication Slot</primary>
+    </indexterm>
+    <indexterm>
+     <primary>Replication Slot</primary>
+    </indexterm>
+    <title>Replication Slots</title>
+    <para>
+     In the context of changeset extraction a replication slot represents a
+     stream of changes which can be replayed to a client in the order they
+     were made on the origin server. Each slot streams a sequence of changes
+     from a single database, sending each change exactly once (unless peeking
+     forward in the stream).
+    </para>
+    <note>
+     <para>PostgreSQL also has streaming replication slots
+     (see <xref linkend="streaming-replication">), but they are used somewhat
+     differently there.
+     </para>
+    </note>
+    <para>
+     Replication slots have an identifier which is unique across all databases
+     in a <productname>PostgreSQL</productname> cluster. Slots persist
+     independently of the connection using them and are crash-safe.  They can
+     be allocated in primary servers and hot-standby servers (streaming
+     replicas or archive-replaying hot standbys), so it's possible to
+     replicate changes over physical replication then read a logical change
+     stream from a physical replica server.
+    </para>
+    <para>
+     Multiple independent slots may exist for a single database. Each slot has
+     its own state, allowing different consumers to receive changes from
+     different points in the database change stream. For most applications a
+     separate slot is required for each changeset consumer.
+    </para>
+    <para>
+     A changeset extraction slot knows nothing about the state of the
+     receiver(s).  It's even possible to have multiple different receivers use
+     the same slot at different times; they'll just get the changes following
+     on from when the last receiver stopped consuming them. Only one receiver
+     may consume changes from a slot at any given time.
+    </para>
+    <sect3 id="changesetextraction-abandoned-slots">
+     <title>Unused/abandoned slots</title>
+     <para>
+      Changeset extraction slots persist across crashes and know nothing about
+      the state of their consumer(s). They will prevent removal of required
+      resources even when there is no connection using them. This consumes
+      storage because neither required WAL nor required rows from the system
+      catalogs can be removed by VACUUM as long as they are required by a
+      replication slot, so if a slot is no longer required it should be
+      dropped.
+     </para>
+     <para>
+      Slots may be created manually, or using a client like a replication
+      tool. Keep this in mind when you retire a client that uses a changeset
+      extraction slot, like a logical replica - you may need to give it a
+      specific command to remove its changeset extraction slot, or drop the
+      slot yourself, even if you did not initially create the slot by hand.
+     </para>
+    </sect3>
+   </sect2>
+   <sect2>
+    <title>Exported Snapshots</title>
+    <para>
+     When a new replication slot is created over the walsender interface a
+     snapshot is exported
+     (see <xref linkend="functions-snapshot-synchronization">) which will show
+     exactly the state of the database after which all changes will be
+     included in the changestream. This can be used to create a new replica by
+     using <link linkend="sql-set-transaction"><literal>SET TRANSACTION
+     SNAPSHOT</literal></link> to read the state of the database at the moment
+     the slot was created. This transaction can then be used to dump the
+     database's state at that point in time which afterwards can be updated
+     using the slot's contents without loosing any changes.
+    </para>
+   </sect2>
+  </sect1>
+  <sect1 id="changesetextraction-walsender">
+   <title>Streaming Replication Protocol Interface</title>
+   <para>
+    The <literal>CREATE_REPLICATION_SLOT SLOT slotname LOGICAL
+    options</literal>, <literal>DROP_REPLICATION_SLOT SLOT slotname</literal>
+    and <literal>START_REPLICATION SLOT slotname LOGICAL options</literal>
+    commands can be used to create, drop and stream changes from a replication
+    slot respectively. These commands are only available over a replication
+    connection; the won't work from the SQL
+    level. See <xref linkend="protocol-replication">.
+   </para>
+   <para>
+    The <command>pg_recvlogical</command> command
+    (see <xref linkend="app-pgrecvlogical">) can be used to control changeset
+    extraction over a walsender connection.
+   </para>
+  </sect1>
+  <sect1 id="changesetextraction-sql">
+   <title>Changeset Extraction <acronym>SQL</acronym> Interface</title>
+   <para>
+     See <xref linkend="functions-replication"> for detailed
+     documentation on the SQL-level API for interacting with changeset
+     extraction.
+   </para>
+   <para>
+    Only replications slots which are consumed over the walsender interface
+    support being used for synchronous replication
+    (see <xref linkend="synchronous-replication">).
+   </para>
+  </sect1>
+  <sect1 id="changesetextraction-catalogs">
+   <title>System catalogs related to changeset extraction</title>
+   <para>
+    The <xref linkend="catalog-pg-replication-slots"> view and the
+    <literal>pg_stat_replication</literal> view
+    in <xref linkend="monitoring-stats-views-table"> provide information about
+    the current state of replication slots and walsender connections
+    respectively. These views apply to both physical and logical replication.
+   </para>
+  </sect1>
+  <sect1 id="changesetextraction-output-plugin">
+   <title>Changeset Extraction Output Plugins</title>
+   <para>
+    An example output plugin can be found in the
+    <link linkend="test-decoding">
+     <filename>contrib/test_decoding</filename>
+    </link>
+    subdirectory of the PostgreSQL source tree.
+   </para>
+   <sect2 id="changesetextraction-output-init">
+    <title>Initialization Function</title>
+    <indexterm zone="changesetextraction">
+     <primary>_PG_output_plugin_init</primary>
+    </indexterm>
+    <para>
+     An output plugin is loaded by dynamically loading a shared library with
+     the output plugin's name as the library basename. To provide the required
+     output plugin callbacks and to indicate that the library is actually an
+     output plugin it needs to provide a function named
+     <function>_PG_output_plugin_init</function>. This function is passed a
+     struct that needs to be filled with the callback function pointers for
+     individual actions.
+     <programlisting>
+typedef struct OutputPluginCallbacks
+{
+    LogicalDecodeStartupCB startup_cb;
+    LogicalDecodeBeginCB begin_cb;
+    LogicalDecodeChangeCB change_cb;
+    LogicalDecodeCommitCB commit_cb;
+    LogicalDecodeShutdownCB shutdown_cb;
+} OutputPluginCallbacks;
+typedef void (*LogicalOutputPluginInit)(struct OutputPluginCallbacks *cb);
+     </programlisting>
+     The <function>begin_cb</function>, <function>change_cb</function>
+     and <function>commit_cb</function> callbacks are required,
+     while <function>startup_cb</function>
+     and <function>shutdown_cb</function> are optional.
+    </para>
+   </sect2>
+   <sect2 id="changesetextraction-output-plugin-callbacks">
+    <title>Output Plugin Callbacks</title>
+    <para>
+     An output plugin gets notified about changes that are happening via
+     various callbacks it needs to provide.
+    </para>
+    <para>
+     Concurrent transactions are decoded in commit order and only changes
+     belonging to a specific transaction are decoded inbetween
+     the <literal>begin</literal> and <literal>commit</literal>
+     callbacks. Transaction that were rolled back explicitly or implicitly
+     will never be
+     decoded. Successfull <link linkend="SQL-SAVEPOINT">SAVEPOINTs</link> are
+     folded into the transaction containing them in the order they were
+     exectuded within that transaction.
+    </para>
+    <sect3 id="changesetextraction-output-plugin-callback-startup">
+     <title>Startup Callback</title>
+     <para>
+      The optional startup callback is called whenever an replication slot
+      is created or asked to stream changes, independent of the number of
+      changes that are ready to be output.
+      <programlisting>
+typedef void (*LogicalDecodeStartupCB) (
+    struct LogicalDecodingContext *ctx,
+    bool is_init
+);
+      </programlisting>
+      The is_init paramter will be true when the replication slot is being
+      created and false otherwise.
+     </para>
+     <para>
+      The startup callback should validate the options present in
+      <literal>ctx-&gt;output_plugin_options</literal>. If the output plugin needs
+      to have state, it can use <literal>ctx-&gt;output_plugin_private</literal> to
+      store it.
+     </para>
+    </sect3>
+    <sect3 id="changesetextraction-output-plugin-shutdown">
+     <title>Shutdown Callback</title>
+     <para>
+      The optional shutdown callback is called whenever a formerly active
+      replication slot is not used anymore and can be used to deallocate
+      resources private to the output plugin. The slot isn't necessarily being
+      dropped, streaming is just being stopped.
+      <programlisting>
+typedef void (*LogicalDecodeShutdownCB) (
+    struct LogicalDecodingContext *ctx
+);
+      </programlisting>
+     </para>
+   </sect3>
+    <sect3 id="changesetextraction-output-plugin-begin">
+     <title>Transaction Begin Callback</title>
+     <para>
+      The required <function>begin_cb</function> callback is called whenever a
+      transaction start has been decoded, but only if we know that the
+      transaction has committed. Aborted transactions and their contents are
+      never decoded.
+      <programlisting>
+typedef void (*LogicalDecodeBeginCB) (
+    struct LogicalDecodingContext *,
+    ReorderBufferTXN *txn
+);
+      </programlisting>
+      The txn parameter contains meta information about the transaction,
+      like the timestamp at which it committed and its xid.
+     </para>
+   </sect3>
+    <sect3 id="changesetextraction-output-plugin-commit">
+     <title>Transaction End Callback</title>
+     <para>
+      The required <function>commit_cb</function> callback is called whenever
+      a transaction commit has been
+      decoded. The <function>change_cb</function> callbacks for all modified
+      rows will have been called before this, if there are have been any
+      modified rows.
+      <programlisting>
+typedef void (*LogicalDecodeCommitCB) (
+    struct LogicalDecodingContext *,
+    ReorderBufferTXN *txn
+);
+      </programlisting>
+     </para>
+    </sect3>
+    <sect3 id="changesetextraction-output-plugin-change">
+     <title>Callback called for each individual change in a
+     transaction</title>
+     <para>
+      The required <function>change_cb</function> callback is called for every
+      individual row modification inside a transaction, be it
+      an <command>INSERT</command>, <command>UPDATE</command>
+      or <command>DELETE</command>. Even if the original command modified
+      several rows at once, the callback will be called indvidually for each
+      row.
+      <programlisting>
+typedef void (*LogicalDecodeChangeCB) (
+    struct LogicalDecodingContext *ctx,
+    ReorderBufferTXN *txn,
+    Relation relation,
+    ReorderBufferChange *change
+);
+      </programlisting>
+      The <parameter>ctx</parameter> and <parameter>txn</parameter> parameters
+      have the same contents as for the <function>begin_cb</function>
+      and <function>commit_cb</function> callbacks, but additionally the
+      relation descriptor <parameter>relation</parameter>for the relation the
+      row belongs to and a struct <parameter>change</parameter> describing the
+      row modification are passed in.
+     </para>
+     <note>
+      <para>
+       Only changes in user defined tables, that are not unlogged
+       (see <xref linkend="SQL-CREATETABLE-UNLOGGED">) or temporary
+       (see <xref linkend="SQL-CREATETABLE-TEMPORARY">)can be extracted using
+       changeset extraction.
+      </para>
+     </note>
+    </sect3>
+   </sect2>
+   <sect2 id="changesetextraction-output-plugin-output">
+    <title>Functions for producing output from an output plugin</title>
+    <para>
+     To actually produce output output plugins can write data to
+     the <literal>StringInfo</literal> output buffer
+     in <literal>ctx-&gt;out</literal> when inside
+     the <function>begin_cb</function>, <function>commit_cb</function>, <function>change_cb</function>
+     callbacks. Before writing to the output
+     buffer <function>OutputPluginPrepareWrite(ctx, last_write)</function> has
+     to be called, and after finishing writing to the
+     buffer <function>OutputPluginWrite(ctx, last_write)</function> has to be
+     called to perform the write. The <parameter>last_write</parameter>
+     indicates whether a particular write was the callback's last write.
+    </para>
+    <para>
+     The following example shows how to output data to the consumer of an
+     output plugin:
+     <programlisting>
+OutputPluginPrepareWrite(ctx, true);
+appendStringInfo(ctx->out, "BEGIN %u", txn->xid);
+OutputPluginWrite(ctx, true);
+     </programlisting>
+    </para>
+   </sect2>
+  </sect1>
+  <sect1 id="changesetextraction-writer">
+   <title>Changeset Extraction Output Writers</title>
+   <para>
+    It is possible to add additional output methods, in addition to the SQL
+    and replication protocol variants, of consuming changeset extraction
+    data. For details look at the implementation of the SQL interface
+    functions
+    in <filename>src/backend/replication/logical/logicalfuncs.c</filename>.
+    Essentially three functions need to be provided, one to read WAL, one to
+    prepare writing output and one to write the output
+    (see <xref linkend="changesetextraction-output-plugin-output">).
+   </para>
+  </sect1>
+  <sect1 id="changesetextraction-synchronous">
+   <title>Synchronous replication support for Changeset Extraction</title>
+   <para>
+    The Changeset Extraction support in <productname>PostgreSQL</productname>
+    supports being used to to
+    build <link linkend="synchronous-replication">synchronous
+    replication</link> solutions, with the same user interface as synchronous
+    replication for <link linkend="streaming-replication">streaming
+    replication</link> if the walsender interface
+    (see <xref linkend="changesetextraction-walsender">) is used to stream out
+    data. Clients have to send <literal>Standby status update (F)</literal>
+    (see <xref linkend="protocol-replication">), just like streaming
+    replication clients do.
+   </para>
+  </sect1>
+ </chapter>
diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml
index 09de4bd..05bed2b 100644
--- a/doc/src/sgml/filelist.sgml
+++ b/doc/src/sgml/filelist.sgml
@@ -91,6 +91,7 @@
 <!ENTITY nls        SYSTEM "nls.sgml">
 <!ENTITY plhandler  SYSTEM "plhandler.sgml">
 <!ENTITY fdwhandler SYSTEM "fdwhandler.sgml">
+<!ENTITY changesetextraction SYSTEM "changesetextraction.sgml">
 <!ENTITY protocol   SYSTEM "protocol.sgml">
 <!ENTITY sources    SYSTEM "sources.sgml">
 <!ENTITY storage    SYSTEM "storage.sgml">
@@ -145,6 +146,7 @@
 <!ENTITY tcn             SYSTEM "tcn.sgml">
 <!ENTITY test-parser     SYSTEM "test-parser.sgml">
 <!ENTITY test-shm-mq     SYSTEM "test-shm-mq.sgml">
+<!ENTITY test-decoding   SYSTEM "test-decoding.sgml">
 <!ENTITY tsearch2        SYSTEM "tsearch2.sgml">
 <!ENTITY unaccent      SYSTEM "unaccent.sgml">
 <!ENTITY uuid-ossp       SYSTEM "uuid-ossp.sgml">
diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml
index be548d7..d4e84ad 100644
--- a/doc/src/sgml/func.sgml
+++ b/doc/src/sgml/func.sgml
@@ -16301,8 +16301,9 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup());
 
    <para>
     PostgreSQL exposes a number of functions for controlling and interacting
-    with replication features. See <xref linkend="streaming-replication">
-    and <xref linkend="streaming-replication-slots">.
+    with replication features. See <xref linkend="streaming-replication">,
+    <xref linkend="streaming-replication-slots">
+    and <xref linkend="changesetextraction">.
    </para>
 
    <para>
@@ -16361,9 +16362,98 @@ postgres=# SELECT * FROM pg_xlogfile_name_offset(pg_stop_backup());
         command <literal>DROP_REPLICATION_SLOT</>.
        </entry>
       </row>
+
+      <row>
+       <entry>
+        <indexterm>
+         <primary>pg_create_decoding_replication_slot</primary>
+        </indexterm>
+        <literal><function>pg_create_decoding_replication_slot(<parameter>slotname</parameter> <type>text</type>, <parameter>plugin</parameter> <type>text</type>)</function></literal>
+       </entry>
+       <entry>
+        (<parameter>slotname</parameter> <type>text</type>, <parameter>xlog_position</parameter> <type>text</type>)
+       </entry>
+       <entry>
+        Creates a new logical (decoding) replication slot named
+        <parameter>slotname</parameter> using the output plugin
+        <parameter>plugin</parameter>. Output plugins are listed amongst the
+        extensions in <literal>pg_catalog.pg_available_extensions</literal>,
+        but there is no specific listing of only output plugins. Same as
+        walsender protocol command <literal>CREATE REPLICATION SLOT ... LOGICAL</literal>.
+       </entry>
+      </row>
+
+      <row>
+       <entry>
+        <indexterm>
+         <primary>pg_decoding_slot_get_changes</primary>
+        </indexterm>
+        <literal><function>pg_decoding_slot_get_changes(<parameter>slotname</parameter> <type>text</type>)</function></literal>
+       </entry>
+       <entry>
+        (<parameter>location</parameter> <type>text</type>, <parameter>xid</parameter> <type>xid</type>, <parameter>data</parameter> <type>text</type>)
+       </entry>
+       <entry>
+        Returns all changes in the slot <parameter>slotname</parameter> since
+        changes have been consumed last, up until the changes visible at the
+        time this call starts. Changes will be consumed.
+       </entry>
+      </row>
+
+      <row>
+       <entry>
+        <indexterm>
+         <primary>pg_decoding_slot_peek_changes</primary>
+        </indexterm>
+        <literal><function>pg_decoding_slot_peek_changes(<parameter>slotname</parameter> <type>text</type>)</function></literal>
+       </entry>
+       <entry>
+        (<parameter>location</parameter> <type>text</type>, <parameter>xid</parameter> <type>xid</type>, <parameter>data</parameter> <type>text</type>)
+       </entry>
+       <entry>
+        Returns all changes in the slot <parameter>slotname</parameter> since
+        changes have been consumed last, up to the changes visible at the start
+        of this call. Changes will not be consumed.
+       </entry>
+      </row>
+
+      <row>
+       <entry>
+        <indexterm>
+         <primary>pg_decoding_slot_get_binary_changes</primary>
+        </indexterm>
+        <literal><function>pg_decoding_slot_get_binary_changes(<parameter>slotname</parameter> <type>text</type>)</function></literal>
+       </entry>
+       <entry>
+        (<parameter>location</parameter> <type>text</type>, <parameter>xid</parameter> <type>xid</type>, <parameter>data</parameter> <type>bytea</type>)
+       </entry>
+       <entry>
+        Returns all changes in the slot <parameter>slotname</parameter> since
+        changes have been consumed last in binary format. Changes will be
+        consumed.
+       </entry>
+      </row>
+
+      <row>
+       <entry>
+        <indexterm>
+         <primary>pg_decoding_slot_peek_binary_changes</primary>
+        </indexterm>
+        <literal><function>pg_decoding_slot_peek_binary_changes(<parameter>slotname</parameter> <type>text</type>)</function></literal>
+       </entry>
+       <entry>
+        (<parameter>location</parameter> <type>text</type>, <parameter>xid</parameter> <type>xid</type>, <parameter>data</parameter> <type>binary</type>)
+       </entry>
+       <entry>
+        Returns all changes in the slot <parameter>slotname</parameter> since
+        changes have been consumed last in binary format. Changes will not
+        be consumed.
+       </entry>
+      </row>
      </tbody>
     </tgroup>
    </table>
+
   </sect2>
 
   <sect2 id="functions-admin-dbobject">
diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml
index b47bf52..dd1709f 100644
--- a/doc/src/sgml/postgres.sgml
+++ b/doc/src/sgml/postgres.sgml
@@ -219,6 +219,7 @@
 
   &spi;
   &bgworker;
+  &changesetextraction;
 
  </part>
 
diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml
index 8d890ba..e53cc38 100644
--- a/doc/src/sgml/protocol.sgml
+++ b/doc/src/sgml/protocol.sgml
@@ -1301,24 +1301,48 @@
 <title>Streaming Replication Protocol</title>
 
 <para>
-To initiate streaming replication, the frontend sends the
-<literal>replication</> parameter in the startup message. A boolean value
-of <literal>true</> tells the backend to go into walsender mode, wherein a
-small set of replication commands can be issued instead of SQL statements. Only
-the simple query protocol can be used in walsender mode.
-Passing a <literal>database</> as the value instructs walsender to connect to
-the database specified in the <literal>dbname</> paramter which will in future
-allow some additional commands to the ones specified below to be run.
+ To initiate streaming replication, the frontend sends the
+ <literal>replication</> parameter in the startup message. This accepts the
+ values <literal>true</literal>, <literal>false</literal> (and their 1/0
+ numeric equivalents) or in 9.4 and above, <literal>database</literal>.
+</para>
+<para>
+ Any valid non-false value tells the backend to go into walsender mode, wherein
+ the small set of replication commands documented below can be issued instead
+ of SQL statements.  Only the simple query protocol can be used in walsender
+ mode.
+</para>
+<para>
+ If <literal>replication</literal> is <literal>true</literal>, no specific
+ database is connected to. In this mode only a subset of walsender subcommands
+ are available. To use database-specific walsender commands like the logical
+ replication commands, <literal>replication</> must be set to
+ <literal>database</> and the name of a database to connect to must be
+ specified in the <literal>database</> startup packet field.
+</para>
+<para>
+ For the purpose of testing replication commands, you can make a replication
+ connection via <application>psql</application> or any other <literal>libpq</literal>-using
+ tool with a connection string including the <literal>replication</literal> option,
+ e.g.:
+ <programlisting>
+  psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;"
+ </programlisting>
+ However it is usually more useful to use
+ <application>pg_receivexlog</application> (for physical replication) or
+ <application>pg_recvlogical</application> (for logical replication).
+</para>
 
+<para>
 The commands accepted in walsender mode are:
-
 <variablelist>
   <varlistentry>
     <term>IDENTIFY_SYSTEM</term>
     <listitem>
      <para>
-      Requests the server to identify itself. Server replies with a result
-      set of a single row, containing four fields:
+      Requests the server to identify itself. Does not require a database name
+      to be specified for the connection. Server replies with a result set of a
+      single row, containing four fields:
      </para>
 
      <para>
@@ -1381,7 +1405,8 @@ The commands accepted in walsender mode are:
     <listitem>
      <para>
       Requests the server to send over the timeline history file for timeline
-      <replaceable class="parameter">tli</replaceable>.  Server replies with a
+      <replaceable class="parameter">tli</replaceable>. Does not require a database
+      name to be specified for the connection. Server replies with a
       result set of a single row, containing two fields:
      </para>
 
@@ -1764,17 +1789,55 @@ The commands accepted in walsender mode are:
      </para>
     </listitem>
   </varlistentry>
+  <varlistentry>
+    <term><literal>START_REPLICATION</literal> <literal>SLOT</literal> <replaceable class="parameter">slotname</> <literal>LOGICAL</literal> <replaceable class="parameter">XXX/XXX</></term>
+    <listitem>
+     <para>
+      Instructs server to start streaming WAL for logical replication, starting
+      at WAL position <replaceable class="parameter">XXX/XXX</>. The server can
+      reply with an error, e.g. if the requested section of WAL has already
+      been recycled. On success, server responds with a CopyBothResponse
+      message, and then starts to stream WAL to the frontend.
+     </para>
+     <para>
+      The output plugin associated with the selected slot is used
+      to process the output for streaming.
+     </para>
+     <variablelist>
+      <varlistentry>
+       <term><literal>SLOT</literal> <replaceable class="parameter">slotname</></term>
+       <listitem>
+         <para>
+          The name of the slot to stream changes from. This parameter is required,
+          and must correspond to an existing logical replication slot created
+          with <literal>CREATE_REPLICATION_SLOT</literal> in
+          <literal>LOGICAL</literal> mode.
+         </para>
+       </listitem>
+      </varlistentry>
+      <varlistentry>
+       <term><replaceable class="parameter">XXX/XXX</></term>
+       <listitem>
+        <para>
+         The WAL position to begin streaming at.
+        </para>
+       </listitem>
+      </varlistentry>
+     </variablelist>
+    </listitem>
+  </varlistentry>
 
   <varlistentry>
-    <term><literal>DROP_REPLICATION_SLOT</literal> <replaceable class="parameter">slotname</></term>
+    <term><literal>DROP_REPLICATION_SLOT</literal> <literal>SLOT</literal> <replaceable class="parameter">slotname</></term>
     <listitem>
      <para>
-      Drops a replication slot, freeing any reserved server-side resources. If
-      the slot is currently in use by an active connection, this command fails.
+     Drops a physical or logical replication slot, freeing any reserved server-side
+     resources. If the slot is currently in use by an active connection this command
+     fails.
      </para>
      <variablelist>
       <varlistentry>
-       <term><replaceable class="parameter">slotname</></term>
+       <term><literal>SLOT</literal> <replaceable class="parameter">slotname</></term>
        <listitem>
          <para>
           The name of the slot to drop.
diff --git a/doc/src/sgml/ref/allfiles.sgml b/doc/src/sgml/ref/allfiles.sgml
index ce7a5e3..1b0962c 100644
--- a/doc/src/sgml/ref/allfiles.sgml
+++ b/doc/src/sgml/ref/allfiles.sgml
@@ -183,6 +183,7 @@ Complete list of usable sgml source files in this directory.
 <!ENTITY pgDumpall          SYSTEM "pg_dumpall.sgml">
 <!ENTITY pgIsready          SYSTEM "pg_isready.sgml">
 <!ENTITY pgReceivexlog      SYSTEM "pg_receivexlog.sgml">
+<!ENTITY pgRecvlogical      SYSTEM "pg_recvlogical.sgml">
 <!ENTITY pgResetxlog        SYSTEM "pg_resetxlog.sgml">
 <!ENTITY pgRestore          SYSTEM "pg_restore.sgml">
 <!ENTITY postgres           SYSTEM "postgres-ref.sgml">
diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml
index e0b8a4e..04e8b92 100644
--- a/doc/src/sgml/ref/create_table.sgml
+++ b/doc/src/sgml/ref/create_table.sgml
@@ -137,7 +137,7 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI
 
   <variablelist>
 
-   <varlistentry>
+   <varlistentry id="SQL-CREATETABLE-TEMPORARY">
     <term><literal>TEMPORARY</> or <literal>TEMP</></term>
     <listitem>
      <para>
@@ -171,7 +171,7 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI
     </listitem>
    </varlistentry>
 
-   <varlistentry>
+   <varlistentry id="SQL-CREATETABLE-UNLOGGED">
     <term><literal>UNLOGGED</></term>
     <listitem>
      <para>
diff --git a/doc/src/sgml/ref/pg_recvlogical.sgml b/doc/src/sgml/ref/pg_recvlogical.sgml
new file mode 100644
index 0000000..8fbb698
--- /dev/null
+++ b/doc/src/sgml/ref/pg_recvlogical.sgml
@@ -0,0 +1,295 @@
+<!--
+doc/src/sgml/ref/pg_recvlogical.sgml
+PostgreSQL documentation
+-->
+
+<refentry id="app-pgrecvlogical">
+ <refmeta>
+  <refentrytitle><application>pg_recvlogical</application></refentrytitle>
+  <manvolnum>1</manvolnum>
+  <refmiscinfo>Application</refmiscinfo>
+ </refmeta>
+
+ <refnamediv>
+  <refname>pg_recvlogical</refname>
+  <refpurpose>Control changeset extraction
+  (see <xref linkend="changesetextraction">) replication streams over a
+  walsender connection.</refpurpose>
+ </refnamediv>
+
+ <indexterm zone="app-pgrecvlogical">
+  <primary>pg_recvlogical</primary>
+ </indexterm>
+
+ <refsynopsisdiv>
+  <cmdsynopsis>
+   <command>pg_recvlogical</command>
+   <arg rep="repeat" choice="opt"><option>option</option></arg>
+  </cmdsynopsis>
+ </refsynopsisdiv>
+
+ <refsect1 id="R1-APP-PGRECVLOGICAL-1">
+  <title>Description</title>
+  <para>
+   <command>pg_recvlogical</command> controls changeset extraction replication
+   slots and streams data from such replication slots.
+  </para>
+  <para>
+   It makes a replication-mode connection, so it is subject to the same
+   constraints as <link
+   linkend="app-pgreceivexlog"><application>pg_receivexlog</application></link>,
+   plus those for logical replication (see <xref
+   linkend="changesetextraction">).
+  </para>
+
+ </refsect1>
+
+ <refsect1>
+  <title>Options</title>
+
+   <para>
+    <application>pg_recvlogical</application> runs in one of three modes, which
+    control its primary action:
+
+    <variablelist>
+
+     <varlistentry>
+      <term><option>--create</option></term>
+      <listitem>
+       <para>
+       Create a new logical replication slot with the name specified in
+       <option>--slot</option>, using the output plugin
+       <option>--plugin</option>, then exit. The slot is created for the
+       database given in <option>--dbname</option>.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry>
+      <term><option>--start</option></term>
+      <listitem>
+       <para>
+       Begin streaming changes from the logical replication slot with the name
+       specified in <option>--slot</option>, continuing until terminated with a
+       signal. If the server side change stream ends with a server
+       shutdown / disconnect, retry in a loop unless <option>--no-loop</option>
+       is specified. The stream format is determined by the output plugin
+       specified when the slot was created.
+       </para>
+       <para>
+       You must connect to the same <option>--dbname</option> as the slot was
+       created with to stream changes from a logical replication slot.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry>
+      <term><option>--drop</option></term>
+      <listitem>
+       <para>
+       Drop the replication slot with the name specified in <option>--slot</option>, then exit.
+       </para>
+      </listitem>
+     </varlistentry>
+    </variablelist>
+
+   </para>
+
+   <para>
+    <application>pg_recvlogical</application> supports all the usual
+    <literal>libpq</literal>-based options. These are explained in detail in the
+    documentation for <application>psql</application> and for <literal>libpq</literal>.
+
+    <variablelist>
+ 
+      <varlistentry>
+       <term><option>-U <replaceable>user</replaceable></option></term>
+       <term><option>--username <replaceable>user</replaceable></option></term>
+       <listitem>
+        <para>
+         Username to connect as. Must have a suitable <literal>pg_hba.conf</literal>
+         entry allowing <literal>replication</literal> connections. Defaults to
+         current operating system user name.
+        </para>
+       </listitem>
+      </varlistentry>
+ 
+      <varlistentry>
+       <term><option>-d <replaceable>database</replaceable></option></term>
+       <term><option>--dbname <replaceable>database</replaceable></option></term>
+       <listitem>
+        <para>
+         The database to connect to in <literal>replication</literal> mode; see
+         mode descriptions for details. May be a libpq connstring instead. Defaults
+         to user name.
+        </para>
+       </listitem>
+      </varlistentry>
+ 
+      <varlistentry>
+       <term><option>-h <replaceable>hostname-or-ip</replaceable></option></term>
+       <term><option>--host <replaceable>hostname-or-ip</replaceable></option></term>
+       <listitem>
+        <para>
+         Host or socket to connect to. See psql and libpq documentation.
+        </para>
+       </listitem>
+      </varlistentry>
+ 
+      <varlistentry>
+       <term><option>-p <replaceable>port</replaceable></option></term>
+       <term><option>--port <replaceable>port</replaceable></option></term>
+       <listitem>
+        <para>
+         Port number to connect to. See <application>libpq</application> for an explanation
+         of default port choices when this is not specified.
+        </para>
+       </listitem>
+      </varlistentry>
+ 
+      <varlistentry>
+       <term><option>-w</option></term>
+       <term><option>--no-password</option></term>
+       <listitem>
+        <para>
+         Prevent prompting for a password. Will exit with an error code if a password is
+         required but not available.
+        </para>
+       </listitem>
+      </varlistentry>
+ 
+      <varlistentry>
+       <term><option>-W</option></term>
+       <term><option>--password</option></term>
+       <listitem>
+        <para>
+         Provide a password for this connection. Please use the pgservice file
+         (see <xref linkend="libpq-pgservice">) or an environment variable
+         instead of this option.
+        </para>
+       </listitem>
+      </varlistentry>
+
+     </variablelist>
+
+   </para>
+
+   <para>
+    The following command-line options control the location and format of the
+    output and other replication behaviour:
+
+    <variablelist>
+
+     <varlistentry>
+      <term><option>-f <replaceable>filename</replaceable></option></term>
+      <term><option>--file=<replaceable>filename</replaceable></option></term>
+      <listitem>
+       <para>
+        Receive decoded transaction data into this file. Use <literal>-</> for stdout.
+       </para>
+      </listitem>
+     </varlistentry>
+
+
+     <varlistentry>
+      <term><option>-n</option></term>
+      <term><option>--no-loop=</option></term>
+      <listitem>
+       <para>
+        When the connection to the server is lost, do not retry in a loop, just exit.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry>
+      <term><option>-P <replaceable>plugin</replaceable></option></term>
+      <term><option>--plugin=<replaceable>plugin</replaceable></option></term>
+      <listitem>
+       <para>
+        When creating a slot, use the specified changeset decoding output
+        plugin. See <xref linkend="changesetextraction">. This option has no
+        effect if the slot already exists.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry>
+      <term><option>-s <replaceable>interval_seconds</replaceable></option></term>
+      <term><option>--status-interval=<replaceable>interval_seconds</replaceable></option></term>
+      <listitem>
+       <para>
+        This option has the same effect as the option of the same name in <link
+        linkend="app-pgreceivexlog"><application>pg_receivexlog</application></link>.
+        See the description there.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry>
+      <term><option>-S <replaceable>slot_name</replaceable></option></term>
+      <term><option>--slot=<replaceable>slot_name</replaceable></option></term>
+      <listitem>
+       <para>
+        In <option>--start</option> mode, use the existing logical replication slot named
+        <replaceable>slot_name</replaceable>. In <option>--create</option> mode, create the
+        slot with this name. In <option>--drop</option> mode, delete the slot with this name.
+       </para>
+      </listitem>
+     </varlistentry>
+
+     <varlistentry>
+      <term><option>-I <replaceable>lsn</replaceable></option></term>
+      <term><option>--startpos=<replaceable>lsn</replaceable></option></term>
+      <listitem>
+       <para>
+        In <option>--start</option> mode, start replication from the given LSN.
+        For details on the effect of this, see the documentation in <xref linkend="changesetextraction">
+        and <xref linkend="protocol-replication">. Ignored in other modes.
+       </para>
+      </listitem>
+     </varlistentry>
+    </variablelist>
+
+   </para>
+
+   <para>
+
+    The following additional options are available:
+
+    <variablelist>
+
+     <varlistentry>
+       <term><option>-v</></term>
+       <term><option>--verbose</></term>
+       <listitem>
+       <para>
+        Output verbose (detailed) error messages suitable for debugging and fault diagnosis.
+       </para>
+       </listitem>
+     </varlistentry>
+
+     <varlistentry>
+       <term><option>-V</></term>
+       <term><option>--version</></term>
+       <listitem>
+       <para>
+       Print the <application>pg_recvlogical</application> version and exit.
+       </para>
+       </listitem>
+     </varlistentry>
+
+     <varlistentry>
+      <term><option>-?</></term>
+      <term><option>--help</></term>
+       <listitem>
+        <para>
+         Show help about <application>pg_recvlogical</application> command line
+         arguments, and exit.
+        </para>
+       </listitem>
+      </varlistentry>
+
+    </variablelist>
+   </para>
+ </refsect1>
+</refentry>
diff --git a/doc/src/sgml/reference.sgml b/doc/src/sgml/reference.sgml
index 87e8e9e..a6575f5 100644
--- a/doc/src/sgml/reference.sgml
+++ b/doc/src/sgml/reference.sgml
@@ -231,6 +231,7 @@
    &pgDumpall;
    &pgIsready;
    &pgReceivexlog;
+   &pgRecvlogical;
    &pgRestore;
    &psqlRef;
    &reindexdb;
-- 
1.8.5.rc2.dirty

