From 849175e102275cc7d0e104149f31e2ba51c3d73a Mon Sep 17 00:00:00 2001 From: Nikhil Sontakke Date: Mon, 5 Mar 2018 21:53:13 +0530 Subject: [PATCH 5/6] pgoutput output plugin support for logical decoding of 2PC. Includes documentation and test cases. --- doc/src/sgml/logicaldecoding.sgml | 128 ++++++++++++++++++++++++- src/backend/access/transam/twophase.c | 32 +++++++ src/backend/replication/logical/proto.c | 100 ++++++++++++++++++-- src/backend/replication/logical/worker.c | 141 +++++++++++++++++++++++++++- src/backend/replication/pgoutput/pgoutput.c | 84 +++++++++++++++++ src/include/access/twophase.h | 1 + src/include/replication/logicalproto.h | 17 +++- 7 files changed, 490 insertions(+), 13 deletions(-) diff --git a/doc/src/sgml/logicaldecoding.sgml b/doc/src/sgml/logicaldecoding.sgml index 5501eed108..0c9a51ae49 100644 --- a/doc/src/sgml/logicaldecoding.sgml +++ b/doc/src/sgml/logicaldecoding.sgml @@ -384,7 +384,12 @@ typedef struct OutputPluginCallbacks LogicalDecodeBeginCB begin_cb; LogicalDecodeChangeCB change_cb; LogicalDecodeCommitCB commit_cb; + LogicalDecodeAbortCB abort_cb; LogicalDecodeMessageCB message_cb; + LogicalDecodeFilterPrepareCB filter_prepare_cb; + LogicalDecodePrepareCB prepare_cb; + LogicalDecodeCommitPreparedCB commit_prepared_cb; + LogicalDecodeAbortPreparedCB abort_prepared_cb; LogicalDecodeFilterByOriginCB filter_by_origin_cb; LogicalDecodeShutdownCB shutdown_cb; } OutputPluginCallbacks; @@ -454,7 +459,12 @@ CREATE TABLE another_catalog_table(data text) WITH (user_catalog_table = true); never get decoded. Successful savepoints are folded into the transaction containing them in the order they were - executed within that transaction. + executed within that transaction. A transaction that is prepared for + a two-phase commit using PREPARE TRANSACTION will + also be decoded if the output plugin callbacks needed for decoding + them are provided. It is possible that the current transaction which + is being decoded is aborted concurrently via a ROLLBACK PREPARED + command. In that case, the logical decoding will be aborted midways. @@ -550,6 +560,74 @@ typedef void (*LogicalDecodeCommitCB) (struct LogicalDecodingContext *ctx, + + Transaction Prepare Callback + + + The optional prepare_cb callback is called whenever + a transaction which is prepared for two-phase commit has been + decoded. The change_cb callbacks for all modified + rows will have been called before this, if there have been any modified + rows. + +typedef void (*LogicalDecodePrepareCB) (struct LogicalDecodingContext *ctx, + ReorderBufferTXN *txn, + XLogRecPtr prepare_lsn); + + + + + + Commit Prepared Transaction Callback + + + The optional commit_prepared_cb callback is called whenever + a commit prepared transaction has been decoded. The gid field, + which is part of the txn parameter can be used in this + callback. + +typedef void (*LogicalDecodeCommitPreparedCB) (struct LogicalDecodingContext *ctx, + ReorderBufferTXN *txn, + XLogRecPtr commit_lsn); + + + + + + Rollback Prepared Transaction Callback + + + The optional abort_prepared_cb callback is called whenever + a rollback prepared transaction has been decoded. The gid field, + which is part of the txn parameter can be used in this + callback. + +typedef void (*LogicalDecodeAbortPreparedCB) (struct LogicalDecodingContext *ctx, + ReorderBufferTXN *txn, + XLogRecPtr abort_lsn); + + + + + + Transaction Abort Callback + + + The required abort_cb callback is called whenever + a transaction abort has to be initiated. This can happen if we are + decoding a transaction that has been prepared for two-phase commit and + a concurrent rollback happens while we are decoding it. It might make + sense, even before we commence decoding, in such cases to check if the + rollback happened even before we start looking at the changes to + completely avoid the decoding of such transactions. + +typedef void (*LogicalDecodeAbortCB) (struct LogicalDecodingContext *ctx, + ReorderBufferTXN *txn, + XLogRecPtr abort_lsn); + + + + Change Callback @@ -559,12 +637,30 @@ typedef void (*LogicalDecodeCommitCB) (struct LogicalDecodingContext *ctx, an INSERT, UPDATE, or DELETE. Even if the original command modified several rows at once the callback will be called individually for each - row. + row. The change_cb callback may access system or + user catalog tables to aid in the process of outputting the row + modification details. The change_cb call should invoke + LogicalLockTransaction function before such access of + system or user catalog tables. In case of decoding a prepared (but yet + uncommitted) transaction or decoding of an uncommitted transaction, this + function interlocks the decoding activity with simultaneous rollback by + another backend of this very same transaction. The + change_cb should invoke + LogicalUnlockTransaction function immediately after + the catalog tables access. typedef void (*LogicalDecodeChangeCB) (struct LogicalDecodingContext *ctx, ReorderBufferTXN *txn, Relation relation, ReorderBufferChange *change); + + Here's an example of the use of LogicalLockTransaction + and LogicalUnlockTransaction in an output plugin: + + if (!LogicalLockTransaction(txn)) + return; + relation = RelationIdGetRelation(reloid); + LogicalUnlockTransaction(txn); The ctx and txn parameters have the same contents as for the begin_cb @@ -614,6 +710,34 @@ typedef bool (*LogicalDecodeFilterByOriginCB) (struct LogicalDecodingContext *ct + + Prepare Filter Callback + + + The optional filter_prepare_cb callback + is called to determine whether data that is part of the current + two-phase commit transaction should be considered for decode + at this prepare stage or as a regular one-phase transaction at + COMMIT PREPARED time later. To signal that + decoding should be skipped, return true; false otherwise. + +typedef bool (*LogicalDecodeFilterPrepareCB) (struct LogicalDecodingContext *ctx, + ReorderBufferTXN *txn, + TransactionId xid, + const char *gid); + + The ctx parameter has the same contents + as for the other callbacks. The txn parameter + contains meta information about the transaction. The xid + contains the XID because txn can be NULL in some cases. + The gid is the identifier that later identifies this + transaction for COMMIT PREPARED or ROLLBACK PREPARED. + The callback has to provide the same static answer for a given combination of + xid and gid every time it is + called. To signal that decoding should be skipped, return true; false otherwise. + + + Generic Message Callback diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index f3091af385..e2db0ebf77 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -549,6 +549,38 @@ MarkAsPrepared(GlobalTransaction gxact, bool lock_held) ProcArrayAdd(&ProcGlobal->allProcs[gxact->pgprocno]); } +/* + * LookupGXact + * Check if the prepared transaction with the given GID is + * around + */ +bool +LookupGXact(const char *gid) +{ + int i; + + LWLockAcquire(TwoPhaseStateLock, LW_EXCLUSIVE); + + for (i = 0; i < TwoPhaseState->numPrepXacts; i++) + { + GlobalTransaction gxact = TwoPhaseState->prepXacts[i]; + + /* Ignore not-yet-valid GIDs */ + if (!gxact->valid) + continue; + if (strcmp(gxact->gid, gid) != 0) + continue; + + LWLockRelease(TwoPhaseStateLock); + + return true; + } + + LWLockRelease(TwoPhaseStateLock); + + return false; +} + /* * LockGXact * Locate the prepared transaction and mark it busy for COMMIT or PREPARE. diff --git a/src/backend/replication/logical/proto.c b/src/backend/replication/logical/proto.c index 948343e4ae..cae4c72fed 100644 --- a/src/backend/replication/logical/proto.c +++ b/src/backend/replication/logical/proto.c @@ -72,10 +72,11 @@ void logicalrep_write_commit(StringInfo out, ReorderBufferTXN *txn, XLogRecPtr commit_lsn) { - uint8 flags = 0; + uint8 flags = 0; pq_sendbyte(out, 'C'); /* sending COMMIT */ + flags |= LOGICALREP_IS_COMMIT; /* send the flags field (unused for now) */ pq_sendbyte(out, flags); @@ -86,21 +87,106 @@ logicalrep_write_commit(StringInfo out, ReorderBufferTXN *txn, } /* - * Read transaction COMMIT from the stream. + * Write ABORT to the output stream. + */ +void +logicalrep_write_abort(StringInfo out, ReorderBufferTXN *txn, + XLogRecPtr abort_lsn) +{ + uint8 flags = 0; + + pq_sendbyte(out, 'C'); /* sending ABORT flag below */ + + flags |= LOGICALREP_IS_ABORT; + /* send the flags field */ + pq_sendbyte(out, flags); + + /* send fields */ + pq_sendint64(out, abort_lsn); + pq_sendint64(out, txn->end_lsn); + pq_sendint64(out, txn->commit_time); +} + +/* + * Read transaction COMMIT|ABORT from the stream. */ void -logicalrep_read_commit(StringInfo in, LogicalRepCommitData *commit_data) +logicalrep_read_commit(StringInfo in, LogicalRepCommitData *commit_data, + uint8 *flags) { - /* read flags (unused for now) */ - uint8 flags = pq_getmsgbyte(in); + /* read flags */ + uint8 commit_flags = pq_getmsgbyte(in); - if (flags != 0) - elog(ERROR, "unrecognized flags %u in commit message", flags); + if (!(commit_flags & LOGICALREP_COMMIT_MASK)) + elog(ERROR, "unrecognized flags %u in commit|abort message", + commit_flags); /* read fields */ commit_data->commit_lsn = pq_getmsgint64(in); commit_data->end_lsn = pq_getmsgint64(in); commit_data->committime = pq_getmsgint64(in); + + /* set gid to empty */ + commit_data->gid[0] = '\0'; + + *flags = commit_flags; +} + +/* + * Write PREPARE to the output stream. + */ +void +logicalrep_write_prepare(StringInfo out, ReorderBufferTXN *txn, + XLogRecPtr prepare_lsn) +{ + uint8 flags = 0; + + pq_sendbyte(out, 'P'); /* sending PREPARE protocol */ + + if (rbtxn_commit_prepared(txn)) + flags |= LOGICALREP_IS_COMMIT_PREPARED; + else if (rbtxn_rollback_prepared(txn)) + flags |= LOGICALREP_IS_ROLLBACK_PREPARED; + else if (rbtxn_prepared(txn)) + flags |= LOGICALREP_IS_PREPARE; + + if (flags == 0) + elog(ERROR, "unrecognized flags %u in [commit|rollback] prepare message", flags); + + /* send the flags field */ + pq_sendbyte(out, flags); + + /* send fields */ + pq_sendint64(out, prepare_lsn); + pq_sendint64(out, txn->end_lsn); + pq_sendint64(out, txn->commit_time); + + /* send gid */ + pq_sendstring(out, txn->gid); +} + +/* + * Read transaction PREPARE from the stream. + */ +void +logicalrep_read_prepare(StringInfo in, LogicalRepCommitData *commit_data, uint8 *flags) +{ + /* read flags */ + uint8 prep_flags = pq_getmsgbyte(in); + + if (!(prep_flags & LOGICALREP_PREPARE_MASK)) + elog(ERROR, "unrecognized flags %u in prepare message", prep_flags); + + /* read fields */ + commit_data->commit_lsn = pq_getmsgint64(in); + commit_data->end_lsn = pq_getmsgint64(in); + commit_data->committime = pq_getmsgint64(in); + + /* read gid */ + strcpy(commit_data->gid, pq_getmsgstring(in)); + + /* set flags */ + *flags = prep_flags; } /* diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index 04985c9f91..5f0b40e760 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -452,8 +452,9 @@ static void apply_handle_commit(StringInfo s) { LogicalRepCommitData commit_data; + uint8 flags = 0; - logicalrep_read_commit(s, &commit_data); + logicalrep_read_commit(s, &commit_data, &flags); Assert(commit_data.commit_lsn == remote_final_lsn); @@ -467,7 +468,11 @@ apply_handle_commit(StringInfo s) replorigin_session_origin_lsn = commit_data.end_lsn; replorigin_session_origin_timestamp = commit_data.committime; - CommitTransactionCommand(); + if (flags & LOGICALREP_IS_COMMIT) + CommitTransactionCommand(); + else if (flags & LOGICALREP_IS_ABORT) + AbortCurrentTransaction(); + pgstat_report_stat(false); store_flush_position(commit_data.end_lsn); @@ -487,6 +492,132 @@ apply_handle_commit(StringInfo s) pgstat_report_activity(STATE_IDLE, NULL); } +static void +apply_handle_prepare_txn(LogicalRepCommitData *commit_data) +{ + Assert(commit_data->commit_lsn == remote_final_lsn); + /* The synchronization worker runs in single transaction. */ + if (IsTransactionState() && !am_tablesync_worker()) + { + /* End the earlier transaction and start a new one */ + BeginTransactionBlock(); + CommitTransactionCommand(); + StartTransactionCommand(); + /* + * Update origin state so we can restart streaming from correct + * position in case of crash. + */ + replorigin_session_origin_lsn = commit_data->end_lsn; + replorigin_session_origin_timestamp = commit_data->committime; + + PrepareTransactionBlock(commit_data->gid); + CommitTransactionCommand(); + pgstat_report_stat(false); + + store_flush_position(commit_data->end_lsn); + } + else + { + /* Process any invalidation messages that might have accumulated. */ + AcceptInvalidationMessages(); + maybe_reread_subscription(); + } + + in_remote_transaction = false; + + /* Process any tables that are being synchronized in parallel. */ + process_syncing_tables(commit_data->end_lsn); + + pgstat_report_activity(STATE_IDLE, NULL); +} + +static void +apply_handle_commit_prepared_txn(LogicalRepCommitData *commit_data) +{ + /* there is no transaction when COMMIT PREPARED is called */ + ensure_transaction(); + + /* + * Update origin state so we can restart streaming from correct + * position in case of crash. + */ + replorigin_session_origin_lsn = commit_data->end_lsn; + replorigin_session_origin_timestamp = commit_data->committime; + + FinishPreparedTransaction(commit_data->gid, true); + CommitTransactionCommand(); + pgstat_report_stat(false); + + store_flush_position(commit_data->end_lsn); + in_remote_transaction = false; + + /* Process any tables that are being synchronized in parallel. */ + process_syncing_tables(commit_data->end_lsn); + + pgstat_report_activity(STATE_IDLE, NULL); +} + +static void +apply_handle_rollback_prepared_txn(LogicalRepCommitData *commit_data) +{ + + + /* + * Update origin state so we can restart streaming from correct + * position in case of crash. + */ + replorigin_session_origin_lsn = commit_data->end_lsn; + replorigin_session_origin_timestamp = commit_data->committime; + + /* + * During logical decoding, on the apply side, it's possible that a prepared + * transaction got aborted while decoding. In that case, we stop the + * decoding and abort the transaction immediately. However the ROLLBACK + * prepared processing still reaches the subscriber. In that case it's ok + * to have a missing gid + */ + if (LookupGXact(commit_data->gid)) + { + /* there is no transaction when ABORT/ROLLBACK PREPARED is called */ + ensure_transaction(); + FinishPreparedTransaction(commit_data->gid, false); + CommitTransactionCommand(); + } + + pgstat_report_stat(false); + + store_flush_position(commit_data->end_lsn); + in_remote_transaction = false; + + /* Process any tables that are being synchronized in parallel. */ + process_syncing_tables(commit_data->end_lsn); + + pgstat_report_activity(STATE_IDLE, NULL); +} + +/* + * Handle PREPARE message. + */ +static void +apply_handle_prepare(StringInfo s) +{ + LogicalRepCommitData commit_data; + uint8 flags = 0; + + logicalrep_read_prepare(s, &commit_data, &flags); + + if (flags & LOGICALREP_IS_PREPARE) + apply_handle_prepare_txn(&commit_data); + else if (flags & LOGICALREP_IS_COMMIT_PREPARED) + apply_handle_commit_prepared_txn(&commit_data); + else if (flags & LOGICALREP_IS_ROLLBACK_PREPARED) + apply_handle_rollback_prepared_txn(&commit_data); + else + ereport(ERROR, + (errcode(ERRCODE_PROTOCOL_VIOLATION), + errmsg("wrong [commit|rollback] prepare message"))); +} + /* * Handle ORIGIN message. * @@ -884,10 +1015,14 @@ apply_dispatch(StringInfo s) case 'B': apply_handle_begin(s); break; - /* COMMIT */ + /* COMMIT|ABORT */ case 'C': apply_handle_commit(s); break; + /* [COMMIT|ROLLBACK] PREPARE */ + case 'P': + apply_handle_prepare(s); + break; /* INSERT */ case 'I': apply_handle_insert(s); diff --git a/src/backend/replication/pgoutput/pgoutput.c b/src/backend/replication/pgoutput/pgoutput.c index d538f25ede..2c04766888 100644 --- a/src/backend/replication/pgoutput/pgoutput.c +++ b/src/backend/replication/pgoutput/pgoutput.c @@ -37,11 +37,21 @@ static void pgoutput_begin_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn); static void pgoutput_commit_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, XLogRecPtr commit_lsn); +static void pgoutput_abort_txn(LogicalDecodingContext *ctx, + ReorderBufferTXN *txn, XLogRecPtr abort_lsn); static void pgoutput_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, Relation rel, ReorderBufferChange *change); static bool pgoutput_origin_filter(LogicalDecodingContext *ctx, RepOriginId origin_id); +static bool pgoutput_filter_prepare(LogicalDecodingContext *ctx, + ReorderBufferTXN *txn, TransactionId xid, const char *gid); +static void pgoutput_prepare_txn(LogicalDecodingContext *ctx, + ReorderBufferTXN *txn, XLogRecPtr prepare_lsn); +static void pgoutput_commit_prepared_txn(LogicalDecodingContext *ctx, + ReorderBufferTXN *txn, XLogRecPtr prepare_lsn); +static void pgoutput_abort_prepared_txn(LogicalDecodingContext *ctx, + ReorderBufferTXN *txn, XLogRecPtr prepare_lsn); static bool publications_valid; @@ -79,6 +89,13 @@ _PG_output_plugin_init(OutputPluginCallbacks *cb) cb->begin_cb = pgoutput_begin_txn; cb->change_cb = pgoutput_change; cb->commit_cb = pgoutput_commit_txn; + cb->abort_cb = pgoutput_abort_txn; + + cb->filter_prepare_cb = pgoutput_filter_prepare; + cb->prepare_cb = pgoutput_prepare_txn; + cb->commit_prepared_cb = pgoutput_commit_prepared_txn; + cb->abort_prepared_cb = pgoutput_abort_prepared_txn; + cb->filter_by_origin_cb = pgoutput_origin_filter; cb->shutdown_cb = pgoutput_shutdown; } @@ -251,6 +268,61 @@ pgoutput_commit_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, OutputPluginWrite(ctx, true); } +/* + * ABORT callback + */ +static void +pgoutput_abort_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, + XLogRecPtr abort_lsn) +{ + OutputPluginUpdateProgress(ctx); + + OutputPluginPrepareWrite(ctx, true); + logicalrep_write_abort(ctx->out, txn, abort_lsn); + OutputPluginWrite(ctx, true); +} + +/* + * PREPARE callback + */ +static void +pgoutput_prepare_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, + XLogRecPtr prepare_lsn) +{ + OutputPluginUpdateProgress(ctx); + + OutputPluginPrepareWrite(ctx, true); + logicalrep_write_prepare(ctx->out, txn, prepare_lsn); + OutputPluginWrite(ctx, true); +} + +/* + * COMMIT PREPARED callback + */ +static void +pgoutput_commit_prepared_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, + XLogRecPtr prepare_lsn) +{ + OutputPluginUpdateProgress(ctx); + + OutputPluginPrepareWrite(ctx, true); + logicalrep_write_prepare(ctx->out, txn, prepare_lsn); + OutputPluginWrite(ctx, true); +} +/* + * PREPARE callback + */ +static void +pgoutput_abort_prepared_txn(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, + XLogRecPtr prepare_lsn) +{ + OutputPluginUpdateProgress(ctx); + + OutputPluginPrepareWrite(ctx, true); + logicalrep_write_prepare(ctx->out, txn, prepare_lsn); + OutputPluginWrite(ctx, true); +} + /* * Sends the decoded DML over wire. */ @@ -364,6 +436,18 @@ pgoutput_change(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, MemoryContextReset(data->context); } +/* + * Filter out unnecessary two-phase transactions. + * + * Currently, we forward all two-phase transactions + */ +static bool +pgoutput_filter_prepare(LogicalDecodingContext *ctx, ReorderBufferTXN *txn, + TransactionId xid, const char *gid) +{ + return false; +} + /* * Currently we always forward. */ diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h index f05cde202f..5a4da6efab 100644 --- a/src/include/access/twophase.h +++ b/src/include/access/twophase.h @@ -44,6 +44,7 @@ extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid, extern void StartPrepare(GlobalTransaction gxact); extern void EndPrepare(GlobalTransaction gxact); extern bool StandbyTransactionIdIsPrepared(TransactionId xid); +extern bool LookupGXact(const char *gid); extern TransactionId PrescanPreparedTransactions(TransactionId **xids_p, int *nxids_p); diff --git a/src/include/replication/logicalproto.h b/src/include/replication/logicalproto.h index 0eb21057c5..886025f3aa 100644 --- a/src/include/replication/logicalproto.h +++ b/src/include/replication/logicalproto.h @@ -69,11 +69,20 @@ typedef struct LogicalRepBeginData TransactionId xid; } LogicalRepBeginData; +#define LOGICALREP_IS_COMMIT 0x01 +#define LOGICALREP_IS_ABORT 0x02 +#define LOGICALREP_IS_PREPARE 0x04 +#define LOGICALREP_IS_COMMIT_PREPARED 0x08 +#define LOGICALREP_IS_ROLLBACK_PREPARED 0x10 +#define LOGICALREP_COMMIT_MASK (LOGICALREP_IS_COMMIT | LOGICALREP_IS_ABORT) +#define LOGICALREP_PREPARE_MASK (LOGICALREP_IS_PREPARE | LOGICALREP_IS_COMMIT_PREPARED | LOGICALREP_IS_ROLLBACK_PREPARED) typedef struct LogicalRepCommitData { + uint8 flag; XLogRecPtr commit_lsn; XLogRecPtr end_lsn; TimestampTz committime; + char gid[GIDSIZE]; } LogicalRepCommitData; extern void logicalrep_write_begin(StringInfo out, ReorderBufferTXN *txn); @@ -81,8 +90,14 @@ extern void logicalrep_read_begin(StringInfo in, LogicalRepBeginData *begin_data); extern void logicalrep_write_commit(StringInfo out, ReorderBufferTXN *txn, XLogRecPtr commit_lsn); +extern void logicalrep_write_abort(StringInfo out, ReorderBufferTXN *txn, + XLogRecPtr abort_lsn); +extern void logicalrep_write_prepare(StringInfo out, ReorderBufferTXN *txn, + XLogRecPtr prepare_lsn); extern void logicalrep_read_commit(StringInfo in, - LogicalRepCommitData *commit_data); + LogicalRepCommitData *commit_data, uint8 *flags); +extern void logicalrep_read_prepare(StringInfo in, + LogicalRepCommitData *commit_data, uint8 *flags); extern void logicalrep_write_origin(StringInfo out, const char *origin, XLogRecPtr origin_lsn); extern char *logicalrep_read_origin(StringInfo in, XLogRecPtr *origin_lsn); -- 2.14.3 (Apple Git-98)