From c6624619c861b723952d025a519b7f3094ca4aa1 Mon Sep 17 00:00:00 2001 From: Peter Smith Date: Sun, 7 Mar 2021 11:47:29 +1100 Subject: [PATCH v51] Fix apply worker empty prepare. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit By sad timing of apply/tablesync workers it is possible to have a "consistent snapshot" that spans prepare/commit in such a way that the tablesync did not do the prepare (because snapshot not consistent) and the apply worker does the begin prepare (‘b’) but it skips all the prepared operations [e.g. inserts] while the tablesync was still busy (see the condition of should_apply_changes_for_rel). This can lead to an "empty prepare", because later when the apply worker does the commit prepare (‘K’), there is nothing in it (the inserts were skipped earlier). This patch implements a two-part fix as suggested [1] on hackers. Part 1 - The begin_prepare handler of apply will always wait for any busy tablesync workers to acheive SYNCDONE/READY state. Part 2 - If (after Part 1) the apply-worker's prepare is found to be lagging behind any of the sync-workers then the subsequent prepared operations will be spooled to a file to be replayed at commit_prepared time. Discussion: [1] https://www.postgresql.org/message-id/CAA4eK1L%3DdhuCRvyDvrXX5wZgc7s1hLRD29CKCK6oaHtVCPgiFA%40mail.gmail.com --- src/backend/replication/logical/tablesync.c | 178 ++++++-- src/backend/replication/logical/worker.c | 634 +++++++++++++++++++++++++++- src/include/replication/worker_internal.h | 3 + src/tools/pgindent/typedefs.list | 2 + 4 files changed, 781 insertions(+), 36 deletions(-) diff --git a/src/backend/replication/logical/tablesync.c b/src/backend/replication/logical/tablesync.c index 50c3ea7..97fc399 100644 --- a/src/backend/replication/logical/tablesync.c +++ b/src/backend/replication/logical/tablesync.c @@ -116,6 +116,9 @@ #include "utils/snapmgr.h" static bool table_states_valid = false; +static List *table_states_not_ready = NIL; +static List *table_states_all = NIL; +static void FetchTableStates(bool *started_tx); StringInfo copybuf = NULL; @@ -359,7 +362,6 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) Oid relid; TimestampTz last_start_time; }; - static List *table_states = NIL; static HTAB *last_start_times = NULL; ListCell *lc; bool started_tx = false; @@ -367,42 +369,14 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) Assert(!IsTransactionState()); /* We need up-to-date sync state info for subscription tables here. */ - if (!table_states_valid) - { - MemoryContext oldctx; - List *rstates; - ListCell *lc; - SubscriptionRelState *rstate; - - /* Clean the old list. */ - list_free_deep(table_states); - table_states = NIL; - - StartTransactionCommand(); - started_tx = true; - - /* Fetch all non-ready tables. */ - rstates = GetSubscriptionNotReadyRelations(MySubscription->oid); - - /* Allocate the tracking info in a permanent memory context. */ - oldctx = MemoryContextSwitchTo(CacheMemoryContext); - foreach(lc, rstates) - { - rstate = palloc(sizeof(SubscriptionRelState)); - memcpy(rstate, lfirst(lc), sizeof(SubscriptionRelState)); - table_states = lappend(table_states, rstate); - } - MemoryContextSwitchTo(oldctx); - - table_states_valid = true; - } + FetchTableStates(&started_tx); /* * Prepare a hash table for tracking last start times of workers, to avoid * immediate restarts. We don't need it if there are no tables that need * syncing. */ - if (table_states && !last_start_times) + if (table_states_not_ready && !last_start_times) { HASHCTL ctl; @@ -416,7 +390,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) * Clean up the hash table when we're done with all tables (just to * release the bit of memory). */ - else if (!table_states && last_start_times) + else if (!table_states_not_ready && last_start_times) { hash_destroy(last_start_times); last_start_times = NULL; @@ -425,7 +399,7 @@ process_syncing_tables_for_apply(XLogRecPtr current_lsn) /* * Process all tables that are being synchronized. */ - foreach(lc, table_states) + foreach(lc, table_states_not_ready) { SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc); @@ -1137,3 +1111,141 @@ copy_table_done: wait_for_worker_state_change(SUBREL_STATE_CATCHUP); return slotname; } + +/* + * Common code to fetch the up-to-date sync state info into the static lists. + */ +static void +FetchTableStates(bool *started_tx) +{ + *started_tx = false; + + if (!table_states_valid) + { + MemoryContext oldctx; + List *rstates; + ListCell *lc; + SubscriptionRelState *rstate; + + /* Clean the old lists. */ + list_free_deep(table_states_all); + table_states_all = NIL; + list_free_deep(table_states_not_ready); + table_states_not_ready = NIL; + + StartTransactionCommand(); + *started_tx = true; + + /* Fetch all tables. */ + rstates = GetSubscriptionRelations(MySubscription->oid); + + /* Allocate the tracking info in a permanent memory context. */ + oldctx = MemoryContextSwitchTo(CacheMemoryContext); + foreach(lc, rstates) + { + SubscriptionRelState *cur_rstate = (SubscriptionRelState *) lfirst(lc); + + /* List of all states */ + rstate = palloc(sizeof(SubscriptionRelState)); + memcpy(rstate, cur_rstate, sizeof(SubscriptionRelState)); + table_states_all = lappend(table_states_all, rstate); + + /* List of only not-ready states */ + if (cur_rstate->state != SUBREL_STATE_READY) + { + rstate = palloc(sizeof(SubscriptionRelState)); + memcpy(rstate, cur_rstate, sizeof(SubscriptionRelState)); + table_states_not_ready = lappend(table_states_not_ready, rstate); + } + } + MemoryContextSwitchTo(oldctx); + + table_states_valid = true; + } +} + +/* + * Are there any tablesyncs which have still not yet reached SYNCDONE/READY state? + */ +bool +AnyTablesyncInProgress() +{ + bool found_busy = false; + bool started_tx = false; + int count = 0; + ListCell *lc; + + /* We need up-to-date sync state info for subscription tables here. */ + FetchTableStates(&started_tx); + + /* + * Process all not-READY tables to see if any are also not-SYNCDONE + */ + foreach(lc, table_states_not_ready) + { + SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc); + + count++; + elog(DEBUG1, + "AnyTablesyncInProgress?: #%d. Table relid %u has state '%c'", + count, + rstate->relid, + rstate->state); + + /* + * When the process_syncing_tables_for_apply changes the state + * from SYNCDONE to READY, that change is actually written directly + * into the list element of table_states_not_ready. + * + * So the "table_states_not_ready" list might end up having a READY + * state in it even though there was none when it was initially + * created. This is reason why we need to check for READY below. + */ + if (rstate->state != SUBREL_STATE_SYNCDONE && + rstate->state != SUBREL_STATE_READY) + { + found_busy = true; + break; + } + } + + if (started_tx) + { + CommitTransactionCommand(); + pgstat_report_stat(false); + } + + elog(DEBUG1, + "AnyTablesyncInProgress?: Scanned %d tables, and found busy = %s", + count, + found_busy ? "true" : "false"); + + return found_busy; +} + +/* + * What is the biggest LSN from the all the known tablesyncs? + */ +XLogRecPtr +BiggestTablesyncLSN() +{ + XLogRecPtr biggest_lsn = InvalidXLogRecPtr; + ListCell *lc; + int count = 0; + + foreach(lc, table_states_all) + { + SubscriptionRelState *rstate = (SubscriptionRelState *) lfirst(lc); + + count++; + if (rstate->lsn > biggest_lsn) + biggest_lsn = rstate->lsn; + } + + elog(DEBUG1, + "BiggestTablesyncLSN: Scanned %d tables. Biggest lsn found = %X/%X", + count, + LSN_FORMAT_ARGS(biggest_lsn)); + + return biggest_lsn; +} diff --git a/src/backend/replication/logical/worker.c b/src/backend/replication/logical/worker.c index e5c7afd..89988b8 100644 --- a/src/backend/replication/logical/worker.c +++ b/src/backend/replication/logical/worker.c @@ -209,6 +209,54 @@ static void subxact_info_add(TransactionId xid); static inline void cleanup_subxact_info(void); /* + * The following are for the support of a spoolfile for prepared messages. + */ + +/* + * A Prepare spoolfile hash entry. We create this entry in the psf_hash. This is + * for maintaining a mapping between the name of the prepared spoolfile, and the + * corresponding fileset handles of same. + */ +typedef struct PsfHashEntry +{ + char name[MAXPGPATH]; /* Hash key --- must be first */ + bool allow_delete; /* ok to delete? */ +} PsfHashEntry; + +/* + * Information about the "current" psf spoolfile. + */ +typedef struct PsfFile +{ + char name[MAXPGPATH];/* psf name - same as the HTAB key. */ + bool is_spooling; /* are we currently spooling to this file? */ + File vfd; /* -1 when the file is closed. */ + off_t cur_offset; /* offset for the next write or read. Reset to 0 + * when file is opened. */ +} PsfFile; + +/* + * Hash table for storing the Prepared spoolfile info along with shared fileset. + */ +static HTAB *psf_hash = NULL; + +/* + * Information about the 'current' open spoolfile is only valid when spooling. + * This is flagged as 'is_spooling' only between begin_prepare and prepare. + */ +static PsfFile psf_cur = { .is_spooling = false, .vfd = -1, .cur_offset = 0 }; + +static void prepare_spoolfile_create(char *path); +static void prepare_spoolfile_write(char action, StringInfo s); +static void prepare_spoolfile_close(void); +static void prepare_spoolfile_delete(char *path); +static bool prepare_spoolfile_exists(char *path); +static void prepare_spoolfile_name(char *path, int szpath, Oid subid, char *gid); +static int prepare_spoolfile_replay_messages(char *path, XLogRecPtr final_lsn); +static bool prepare_spoolfile_handler(LogicalRepMsgType action, StringInfo s); +static void prepare_spoolfile_on_proc_exit(int status, Datum arg); + +/* * Serialize and deserialize changes for a toplevel transaction. */ static void stream_cleanup_files(Oid subid, TransactionId xid); @@ -728,6 +776,9 @@ apply_handle_begin_prepare(StringInfo s) { LogicalRepBeginPrepareData begin_data; + /* Tablesync should never receive prepare. */ + Assert(!am_tablesync_worker()); + logicalrep_read_begin_prepare(s, &begin_data); /* @@ -739,6 +790,90 @@ apply_handle_begin_prepare(StringInfo s) errmsg("transaction identifier \"%s\" is already in use", begin_data.gid))); + /* + * By sad timing of apply/tablesync workers it is possible to have a + * “consistent snapshot” that spans prepare/commit in such a way that + * the tablesync did not do the prepare (because snapshot not consistent) + * and the apply worker does the begin prepare (‘b’) but it skips all + * the prepared operations [e.g. inserts] while the tablesync was still + * busy (see the condition of should_apply_changes_for_rel). + * + * This can lead to an "empty prepare", because later when the apply + * worker does the commit prepare (‘K’), there is nothing in it (the + * inserts were skipped earlier). + * + * We avoid this using the 2 part logic: (1) Wait for all tablesync workers + * to reach SYNCDONE/READY state; (2) If the begin_prepare lsn is now + * behind any tablesync lsn then spool the prepared messages to a file + * to be replayed later at commit_prepared time. + * + * ----- + * + * XXX - The 2PC protocol needs the publisher to be aware when the PREPARE + * has been successfully acted on. But because of this "empty prepare" + * case now the prepared messages may be spooled to a file and, when + * that happens the PREPARE would not happen at the usual time, but would + * be deferred until COMMIT PREPARED time. This quirk could only happen + * immediately after the initial table synchronization phase; once all + * tables have acheived READY state the 2PC protocol will behave normally. + * + * A future release may be able to detect when all tables are READY and set + * a flag to indicate this subscription/slot is ready for two_phase + * decoding. Then at the publisher-side, we could enable wait-for-prepares + * only when all the slots of WALSender have that flag set. + */ + if (!am_tablesync_worker()) + { + /* + * Part 1 of 2: + * + * Make sure every tablesync has reached at least SYNCDONE state + * before letting the apply worker proceed. + */ + elog(DEBUG1, + "apply_handle_begin_prepare, end_lsn = %X/%X, final_lsn = %X/%X, lstate_lsn = %X/%X", + LSN_FORMAT_ARGS(begin_data.end_lsn), + LSN_FORMAT_ARGS(begin_data.final_lsn), + LSN_FORMAT_ARGS(MyLogicalRepWorker->relstate_lsn)); + + while (AnyTablesyncInProgress()) + { + process_syncing_tables(begin_data.final_lsn); + + /* This latch is to prevent 100% CPU looping. */ + (void) WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + 1000L, WAIT_EVENT_LOGICAL_SYNC_STATE_CHANGE); + ResetLatch(MyLatch); + } + + /* + * Part 2 of 2: + * + * If (when getting to SYNCDONE/READY state) some tablesync went + * beyond this begin_prepare LSN then set all messages (until + * prepared) will be saved to a spoolfile for replay later at + * commit_prepared time. + */ + if (begin_data.final_lsn < BiggestTablesyncLSN()) + { + char psfpath[MAXPGPATH]; + + /* + * Create the spoolfile. + */ + prepare_spoolfile_name(psfpath, sizeof(psfpath), + MyLogicalRepWorker->subid, begin_data.gid); + prepare_spoolfile_create(psfpath); + + /* + * From now, until the handle_prepare we are spooling to the + * current psf. + */ + psf_cur.is_spooling = true; + } + } + remote_final_lsn = begin_data.final_lsn; in_remote_transaction = true; @@ -754,9 +889,58 @@ apply_handle_prepare(StringInfo s) { LogicalRepPreparedTxnData prepare_data; + /* + * If we were using a psf spoolfile, then write the PREPARE as the final + * message. This prepare information will be used at commit_prepared time. + */ + if (psf_cur.is_spooling) + { + PsfHashEntry *hentry; + + /* Write the PREPARE info to the psf file. */ + prepare_spoolfile_handler(LOGICAL_REP_MSG_PREPARE, s); + + /* + * Flush the spoolfile, so changes can survive a restart. + */ + FileSync(psf_cur.vfd, WAIT_EVENT_DATA_FILE_SYNC); + + /* + * We are finished spooling to the current psf. + */ + psf_cur.is_spooling = false; + + /* + * The commit_prepare will need the spoolfile, so unregister it for + * removal on proc-exit just in case there is an unexpected restart + * between now and when commit_prepared happens. + */ + hentry = (PsfHashEntry *) hash_search(psf_hash, + psf_cur.name, + HASH_FIND, + NULL); + Assert(hentry); + hentry->allow_delete = false; + + /* + * The psf_cur.vfd is meaningful only between begin_prepare and prepared. + * So close it now. Any messages written to the psf will be applied + * later during handle_commit_prepared. + */ + prepare_spoolfile_close(); + + in_remote_transaction = false; + return; + } + logicalrep_read_prepare(s, &prepare_data); - Assert(prepare_data.prepare_lsn == remote_final_lsn); + /* + * Normally, prepare_lsn == remote_final_lsn, but if this prepare message + * was dispatched via the psf spoolfile replay then the remote_final_lsn + * is set to commit lsn instead. Hence the <= instead of == check below. + */ + Assert(prepare_data.prepare_lsn <= remote_final_lsn); if (IsTransactionState()) { @@ -803,9 +987,38 @@ static void apply_handle_commit_prepared(StringInfo s) { LogicalRepPreparedTxnData prepare_data; + char psfpath[MAXPGPATH]; logicalrep_read_commit_prepared(s, &prepare_data); + /* + * If this prepare's messages were being spooled to a file, then replay + * them all now. + */ + prepare_spoolfile_name(psfpath, sizeof(psfpath), + MyLogicalRepWorker->subid, prepare_data.gid); + if (prepare_spoolfile_exists(psfpath)) + { + int nchanges; + + /* + * Replay/dispatch the spooled messages (including lastly, the PREPARE + * message). + */ + + ensure_transaction(); + + nchanges = prepare_spoolfile_replay_messages(psfpath, prepare_data.prepare_lsn); + elog(DEBUG1, + "apply_handle_commit_prepared: replayed %d (all) changes.", + nchanges); + + /* + * After replaying the psf it is no longer needed. Just delete it. + */ + prepare_spoolfile_delete(psfpath); + } + /* there is no transaction when COMMIT PREPARED is called */ ensure_transaction(); @@ -836,15 +1049,37 @@ static void apply_handle_rollback_prepared(StringInfo s) { LogicalRepRollbackPreparedTxnData rollback_data; + bool using_psf; + char psfpath[MAXPGPATH]; logicalrep_read_rollback_prepared(s, &rollback_data); /* + * If this prepare's messages were being spooled to a file, then cleanup + * the file. + */ + prepare_spoolfile_name(psfpath, sizeof(psfpath), + MyLogicalRepWorker->subid, rollback_data.gid); + using_psf = prepare_spoolfile_exists(psfpath); + if (using_psf) + { + /* + * We are finished with this spoolfile. Delete it. + */ + prepare_spoolfile_delete(psfpath); + } + + /* * It is possible that we haven't received prepare because it occurred * before walsender reached a consistent point in which case we need to * skip rollback prepared. + * + * And we also skip the FinishPreparedTransaction if we're using the + * Prepare Spoolfile (using_psf) because in that case there is no matching + * PrepareTransactionBlock done yet. */ - if (LookupGXact(rollback_data.gid, rollback_data.prepare_end_lsn, + if (!using_psf && + LookupGXact(rollback_data.gid, rollback_data.prepare_end_lsn, rollback_data.preparetime)) { /* @@ -884,6 +1119,7 @@ apply_handle_origin(StringInfo s) * remote transaction and before any actual writes. */ if (!in_streamed_transaction && + !psf_cur.is_spooling && (!in_remote_transaction || (IsTransactionState() && !am_tablesync_worker()))) ereport(ERROR, @@ -1302,6 +1538,9 @@ apply_handle_insert(StringInfo s) TupleTableSlot *remoteslot; MemoryContext oldctx; + if (prepare_spoolfile_handler(LOGICAL_REP_MSG_INSERT, s)) + return; + if (handle_streamed_transaction(LOGICAL_REP_MSG_INSERT, s)) return; @@ -1423,6 +1662,9 @@ apply_handle_update(StringInfo s) RangeTblEntry *target_rte; MemoryContext oldctx; + if (prepare_spoolfile_handler(LOGICAL_REP_MSG_UPDATE, s)) + return; + if (handle_streamed_transaction(LOGICAL_REP_MSG_UPDATE, s)) return; @@ -1581,6 +1823,9 @@ apply_handle_delete(StringInfo s) TupleTableSlot *remoteslot; MemoryContext oldctx; + if (prepare_spoolfile_handler(LOGICAL_REP_MSG_DELETE, s)) + return; + if (handle_streamed_transaction(LOGICAL_REP_MSG_DELETE, s)) return; @@ -1950,6 +2195,9 @@ apply_handle_truncate(StringInfo s) List *relids_logged = NIL; ListCell *lc; + if (prepare_spoolfile_handler(LOGICAL_REP_MSG_TRUNCATE, s)) + return; + if (handle_streamed_transaction(LOGICAL_REP_MSG_TRUNCATE, s)) return; @@ -2235,6 +2483,23 @@ LogicalRepApplyLoop(XLogRecPtr last_received) TimeLineID tli; /* + * Initialize the psf_hash table if we haven't yet. This will be used for + * the entire duration of the apply worker so create it in permanent + * context. + */ + if (psf_hash == NULL) + { + HASHCTL hash_ctl; + PsfHashEntry *hentry; + + hash_ctl.keysize = sizeof(hentry->name); + hash_ctl.entrysize = sizeof(PsfHashEntry); + hash_ctl.hcxt = ApplyContext; + psf_hash = hash_create("PrepareSpoolfileHash", 1024, &hash_ctl, + HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + } + + /* * Init the ApplyMessageContext which we clean up after each replication * protocol message. */ @@ -2354,7 +2619,7 @@ LogicalRepApplyLoop(XLogRecPtr last_received) /* confirm all writes so far */ send_feedback(last_received, false, false); - if (!in_remote_transaction && !in_streamed_transaction) + if (!in_remote_transaction && !in_streamed_transaction && !psf_cur.is_spooling) { /* * If we didn't get any transactions for a while there might be @@ -3102,6 +3367,9 @@ ApplyWorkerMain(Datum main_arg) /* Attach to slot */ logicalrep_worker_attach(worker_slot); + /* Arrange to delete any unwanted psf file(s) at proc-exit */ + on_proc_exit(prepare_spoolfile_on_proc_exit, 0); + /* Setup signal handling */ pqsignal(SIGHUP, SignalHandlerForConfigReload); pqsignal(SIGTERM, die); @@ -3279,3 +3547,363 @@ IsLogicalWorker(void) { return MyLogicalRepWorker != NULL; } + +/* + * Handle the PREPARE spoolfile (if any) + * + * It can be necessary to redirect the PREPARE messages to a spoolfile (see + * apply_handle_begin_prepare) and then replay them back at the COMMIT PREPARED + * time. If needed, this is the common function to do that file redirection. + * + * Returns true if the message was redirected to the spoolfile, false + * otherwise (regular mode). + */ +static bool +prepare_spoolfile_handler(LogicalRepMsgType action, StringInfo s) +{ + elog(DEBUG1, + "prepare_spoolfile_handler for action '%c'. %s write to spool file", + action, + psf_cur.is_spooling ? "Do" : "Don't"); + + if (!psf_cur.is_spooling) + return false; + + Assert(!in_streamed_transaction); + + /* write the change to the current file */ + prepare_spoolfile_write(action, s); + + return true; +} + +/* + * Create the spoolfile used to serialize the prepare messages. + */ +static void +prepare_spoolfile_create(char *path) +{ + bool found; + PsfHashEntry *hentry; + + elog(DEBUG1, "creating file \"%s\" for prepare changes", path); + + Assert(!psf_cur.is_spooling); + + /* create or find the prepare spoolfile entry in the psf_hash */ + hentry = (PsfHashEntry *) hash_search(psf_hash, + path, + HASH_ENTER | HASH_FIND, + &found); + + if (!found) + { + elog(DEBUG1, "Not found file \"%s\". Create it.", path); + psf_cur.vfd = PathNameOpenFile(path, O_RDWR | O_CREAT | O_TRUNC | PG_BINARY); + if (psf_cur.vfd < 0) + { + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not create file \"%s\": %m", path))); + } + memcpy(psf_cur.name, path, sizeof(psf_cur.name)); + psf_cur.cur_offset = 0; + hentry->allow_delete = true; + } + else + { + /* + * Open the file and seek to the beginning because we always want to + * create/overwrite this file. + */ + elog(DEBUG1, "Found file \"%s\". Overwrite it.", path); + psf_cur.vfd = PathNameOpenFile(path, O_RDWR | O_CREAT | O_TRUNC | PG_BINARY); + if (psf_cur.vfd < 0) + { + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", path))); + } + memcpy(psf_cur.name, path, sizeof(psf_cur.name)); + psf_cur.cur_offset = 0; + hentry->allow_delete = true; + } + + /* Sanity checks */ + Assert(psf_cur.vfd >= 0); + Assert(psf_cur.cur_offset == 0); + Assert(prepare_spoolfile_exists(path)); +} + +/* + * Close the "current" spoolfile and unset the fd. + */ +static void +prepare_spoolfile_close() +{ + if (psf_cur.vfd >= 0) + FileClose(psf_cur.vfd); + + /* Mark this fd as not valid to use anymore. */ + psf_cur.is_spooling = false; + psf_cur.vfd = -1; + psf_cur.cur_offset = 0; +} + +/* + * Delete the specified psf spoolfile, and any HTAB associated with it. + */ +static void +prepare_spoolfile_delete(char *path) +{ + /* The current psf should be closed already, but make sure anyway. */ + prepare_spoolfile_close(); + + /* Delete the file off the disk. */ + unlink(path); + + /* Remove any entry from the psf_hash, if present */ + hash_search(psf_hash, path, HASH_REMOVE, NULL); +} + +/* + * Serialize a change to the prepare spoolfile for the current toplevel transaction. + * + * The change is serialized in a simple format, with length (not including + * the length), action code (identifying the message type) and message + * contents (without the subxact TransactionId value). + */ +static void +prepare_spoolfile_write(char action, StringInfo s) +{ + int len; + int bytes_written; + + Assert(psf_cur.is_spooling); + + elog(DEBUG1, "prepare_spoolfile_write: writing action '%c'", action); + + /* total on-disk size, including the action type character */ + len = (s->len - s->cursor) + sizeof(char); + + /* first write the size */ + bytes_written = FileWrite(psf_cur.vfd, (char *)&len, sizeof(len), + psf_cur.cur_offset, WAIT_EVENT_DATA_FILE_WRITE); + Assert(bytes_written == sizeof(len)); + psf_cur.cur_offset += bytes_written; + + /* then the action */ + bytes_written = FileWrite(psf_cur.vfd, &action, sizeof(action), + psf_cur.cur_offset, WAIT_EVENT_DATA_FILE_WRITE); + Assert(bytes_written == sizeof(action)); + psf_cur.cur_offset += bytes_written; + + /* and finally the remaining part of the buffer (after the XID) */ + len = (s->len - s->cursor); + + bytes_written = FileWrite(psf_cur.vfd, &s->data[s->cursor], len, + psf_cur.cur_offset, WAIT_EVENT_DATA_FILE_WRITE); + Assert(bytes_written == len); + psf_cur.cur_offset += bytes_written; +} + +/* + * Is there a prepare spoolfile for the specified path? + */ +static bool +prepare_spoolfile_exists(char *path) +{ + bool found; + PsfHashEntry *hentry; + + /* Find the prepare spoolfile entry in the psf_hash */ + hentry = (PsfHashEntry *) hash_search(psf_hash, + path, + HASH_FIND, + &found); + + if (!found) + { + /* + * Hash doesn't know about it, but perhaps the Hash was destroyed by a + * restart, so let's check the file existence on disk. + */ + File fd = PathNameOpenFile(path, O_RDONLY | PG_BINARY); + + found = fd >= 0; + if (fd >= 0) + FileClose(fd); + + /* + * And if it was found on disk then create the HTAB entry for it. + */ + if (found) + { + hentry = (PsfHashEntry *) hash_search(psf_hash, + path, + HASH_ENTER, + NULL); + hentry->allow_delete = false; + } + } + + return found; +} + +/* + * Replay (apply) all the prepared messages that are in the prepare spoolfile. + */ +static int +prepare_spoolfile_replay_messages(char *path, XLogRecPtr final_lsn) +{ + StringInfoData s2; + int nchanges = 0; + char *buffer = NULL; + MemoryContext oldctx, + oldctx2; + PsfFile psf = { .is_spooling = false, .vfd = -1, .cur_offset = 0 }; + + elog(DEBUG1, + "prepare_spoolfile_replay_messages: replaying changes from file \"%s\"", + path); + + /* + * Allocate memory required to process all the messages in + * TopTransactionContext to avoid it getting reset after each message is + * processed. + */ + oldctx = MemoryContextSwitchTo(TopTransactionContext); + + psf.vfd = PathNameOpenFile(path, O_RDONLY | PG_BINARY); + if (psf.vfd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read from prepared spoolfile \"%s\": %m", + path))); + + buffer = palloc(BLCKSZ); + initStringInfo(&s2); + + MemoryContextSwitchTo(oldctx); + + /* + * Make sure the handle apply_dispatch methods are aware we're in a remote + * transaction. + */ + remote_final_lsn = final_lsn; + in_remote_transaction = true; + pgstat_report_activity(STATE_RUNNING, NULL); + + /* + * Read the entries one by one and pass them through the same logic as in + * apply_dispatch. + */ + while (true) + { + int nbytes; + int len; + + CHECK_FOR_INTERRUPTS(); + + /* read length of the on-disk record */ + nbytes = FileRead(psf.vfd, (char *) &len, sizeof(len), + psf.cur_offset, WAIT_EVENT_DATA_FILE_READ); + psf.cur_offset += nbytes; + + /* have we reached end of the file? */ + if (nbytes == 0) + break; + + /* do we have a correct length? */ + if (nbytes != sizeof(len)) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read from prepared spoolfile \"%s\": %m", + path))); + + Assert(len > 0); + + /* make sure we have sufficiently large buffer */ + buffer = repalloc(buffer, len); + + /* and finally read the data into the buffer */ + nbytes = FileRead(psf.vfd, buffer, len, + psf.cur_offset, WAIT_EVENT_DATA_FILE_READ); + psf.cur_offset += nbytes; + if (nbytes != len) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read from prepared spoolfile \"%s\": %m", + path))); + + /* copy the buffer to the stringinfo and call apply_dispatch */ + resetStringInfo(&s2); + appendBinaryStringInfo(&s2, buffer, len); + + /* Ensure we are reading the data into our memory context. */ + oldctx2 = MemoryContextSwitchTo(ApplyMessageContext); + + apply_dispatch(&s2); + + MemoryContextReset(ApplyMessageContext); + + MemoryContextSwitchTo(oldctx2); + + nchanges++; + + if (nchanges % 1000 == 0) + elog(DEBUG1, "replayed %d changes from file '%s'", + nchanges, path); + } + + FileClose(psf.vfd); + + elog(DEBUG1, "replayed %d (all) changes from file \"%s\"", + nchanges, path); + + return nchanges; +} + +/* + * Format the filename for the prepare spoolfile. + */ +static void +prepare_spoolfile_name(char *path, int szpath, Oid subid, char *gid) +{ + PsfHashEntry *hentry; + + /* + * This name is used as the key in the psf_hash HTAB. + * + * Therefore, the name and the key must be exactly same lengths and padded + * with '\0' so garbage does not impact the HTAB lookups. + */ + Assert(sizeof(hentry->name) == MAXPGPATH); + Assert(szpath == MAXPGPATH); + memset(path, '\0', MAXPGPATH); + + snprintf(path, MAXPGPATH, "pg_twophase/%u-%s.prep_changes", subid, gid); +} + +/* + * proc_exit callback to remove unwanted psf files. + */ +static void +prepare_spoolfile_on_proc_exit(int status, Datum arg) +{ + HASH_SEQ_STATUS seq_status; + PsfHashEntry *hentry; + + /* Iterate the HTAB looking for what file can be deleted. */ + if (psf_hash) + { + hash_seq_init(&seq_status, psf_hash); + while ((hentry = (PsfHashEntry *) hash_seq_search(&seq_status)) != NULL) + { + char *path = hentry->name; + + if (hentry->allow_delete) + prepare_spoolfile_delete(path); + } + } +} diff --git a/src/include/replication/worker_internal.h b/src/include/replication/worker_internal.h index 1cac75e..95d78e9 100644 --- a/src/include/replication/worker_internal.h +++ b/src/include/replication/worker_internal.h @@ -86,6 +86,9 @@ extern void ReplicationOriginNameForTablesync(Oid suboid, Oid relid, char *originname, int szorgname); extern char *LogicalRepSyncTableStart(XLogRecPtr *origin_startpos); +extern bool AnyTablesyncInProgress(void); +extern XLogRecPtr BiggestTablesyncLSN(void); + void process_syncing_tables(XLogRecPtr current_lsn); void invalidate_syncing_table_states(Datum arg, int cacheid, uint32 hashvalue); diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 745b51d..4ffcef5 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -1958,6 +1958,8 @@ ProtocolVersion PrsStorage PruneState PruneStepResult +PsfFile +PsfHashEntry PsqlScanCallbacks PsqlScanQuoteType PsqlScanResult -- 1.8.3.1