From cc438a3fb50835d29ea2995e0372a70051be452d Mon Sep 17 00:00:00 2001 From: Masahiko Sawada Date: Mon, 10 May 2021 20:32:25 +0900 Subject: [PATCH v37 3/9] Support two-phase commit for foreign transactions. Co-authored-by: Masahiko Sawada, Ashutosh Bapat --- .../postgres_fdw/expected/postgres_fdw.out | 2 +- src/backend/access/rmgrdesc/Makefile | 1 + src/backend/access/rmgrdesc/fdwxactdesc.c | 59 + src/backend/access/rmgrdesc/xlogdesc.c | 4 +- src/backend/access/transam/Makefile | 2 + src/backend/access/transam/fdwxact.c | 1945 ++++++++++++++++- src/backend/access/transam/fdwxact_launcher.c | 587 +++++ src/backend/access/transam/fdwxact_resolver.c | 339 +++ src/backend/access/transam/rmgr.c | 1 + src/backend/access/transam/twophase.c | 35 + src/backend/access/transam/xact.c | 4 +- src/backend/access/transam/xlog.c | 41 +- src/backend/catalog/dependency.c | 5 +- src/backend/catalog/system_views.sql | 3 + src/backend/commands/dbcommands.c | 14 + src/backend/commands/foreigncmds.c | 78 +- src/backend/foreign/foreign.c | 6 + src/backend/nodes/copyfuncs.c | 15 + src/backend/nodes/equalfuncs.c | 14 + src/backend/parser/gram.y | 29 +- src/backend/postmaster/bgworker.c | 8 + src/backend/postmaster/postmaster.c | 13 +- src/backend/replication/logical/decode.c | 1 + src/backend/replication/syncrep.c | 14 +- src/backend/storage/ipc/ipci.c | 6 + src/backend/storage/ipc/procarray.c | 42 +- src/backend/storage/lmgr/lwlocknames.txt | 2 + src/backend/tcop/postgres.c | 16 + src/backend/tcop/utility.c | 12 +- src/backend/utils/activity/wait_event.c | 15 + src/backend/utils/misc/guc.c | 48 + src/backend/utils/misc/postgresql.conf.sample | 14 + src/bin/initdb/initdb.c | 1 + src/bin/pg_controldata/pg_controldata.c | 2 + src/bin/pg_resetwal/pg_resetwal.c | 2 + src/bin/pg_waldump/rmgrdesc.c | 1 + src/include/access/fdwxact.h | 88 +- src/include/access/fdwxact_launcher.h | 29 + src/include/access/fdwxact_resolver.h | 22 + src/include/access/fdwxact_xlog.h | 49 + src/include/access/resolver_internal.h | 59 + src/include/access/rmgrlist.h | 1 + src/include/access/twophase.h | 1 + src/include/access/xlog_internal.h | 1 + src/include/catalog/pg_control.h | 1 + src/include/catalog/pg_proc.dat | 23 + src/include/commands/defrem.h | 2 + src/include/foreign/fdwapi.h | 2 + src/include/nodes/nodes.h | 1 + src/include/nodes/parsenodes.h | 10 +- src/include/replication/syncrep.h | 2 +- src/include/storage/procarray.h | 1 + src/include/utils/guc_tables.h | 2 + src/include/utils/wait_event.h | 7 +- src/test/regress/expected/rules.out | 8 + 55 files changed, 3623 insertions(+), 67 deletions(-) create mode 100644 src/backend/access/rmgrdesc/fdwxactdesc.c create mode 100644 src/backend/access/transam/fdwxact_launcher.c create mode 100644 src/backend/access/transam/fdwxact_resolver.c create mode 100644 src/include/access/fdwxact_launcher.h create mode 100644 src/include/access/fdwxact_resolver.h create mode 100644 src/include/access/fdwxact_xlog.h create mode 100644 src/include/access/resolver_internal.h diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index 343b33473b..01c43d80ff 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -9309,7 +9309,7 @@ SELECT count(*) FROM ft1; -- error here PREPARE TRANSACTION 'fdw_tpc'; -ERROR: cannot PREPARE a transaction that has operated on foreign tables +ERROR: cannot PREPARE a distributed transaction that has operated on a foreign server not supporting two-phase commit protocol ROLLBACK; WARNING: there is no transaction in progress -- =================================================================== diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile index f88d72fd86..982c1a36cc 100644 --- a/src/backend/access/rmgrdesc/Makefile +++ b/src/backend/access/rmgrdesc/Makefile @@ -13,6 +13,7 @@ OBJS = \ clogdesc.o \ committsdesc.o \ dbasedesc.o \ + fdwxactdesc.o \ genericdesc.o \ gindesc.o \ gistdesc.o \ diff --git a/src/backend/access/rmgrdesc/fdwxactdesc.c b/src/backend/access/rmgrdesc/fdwxactdesc.c new file mode 100644 index 0000000000..4e97486640 --- /dev/null +++ b/src/backend/access/rmgrdesc/fdwxactdesc.c @@ -0,0 +1,59 @@ +/*------------------------------------------------------------------------- + * + * fdwxactdesc.c + * rmgr descriptor routines for access/transam/fdwxact.c + * + * This module describes the WAL records for foreign transaction manager. + * + * Portions Copyright (c) 2021, PostgreSQL Global Development Group + * + * src/backend/access/rmgrdesc/fdwxactdesc.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/fdwxact_xlog.h" + +void +fdwxact_desc(StringInfo buf, XLogReaderState *record) +{ + char *rec = XLogRecGetData(record); + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + if (info == XLOG_FDWXACT_INSERT) + { + FdwXactStateOnDiskData *fdwxact_insert = (FdwXactStateOnDiskData *) rec; + + appendStringInfo(buf, "xid: %u, dbid: %u, umid: %u, serverid: %u, owner: %u, identifier: %s", + fdwxact_insert->xid, + fdwxact_insert->dbid, + fdwxact_insert->umid, + fdwxact_insert->serverid, + fdwxact_insert->owner, + fdwxact_insert->identifier); + } + else + { + xl_fdwxact_remove *fdwxact_remove = (xl_fdwxact_remove *) rec; + + appendStringInfo(buf, "xid: %u, umid: %u", + fdwxact_remove->xid, + fdwxact_remove->umid); + } + +} + +const char * +fdwxact_identify(uint8 info) +{ + switch (info & ~XLR_INFO_MASK) + { + case XLOG_FDWXACT_INSERT: + return "NEW FOREIGN TRANSACTION"; + case XLOG_FDWXACT_REMOVE: + return "REMOVE FOREIGN TRANSACTION"; + } + /* Keep compiler happy */ + return NULL; +} diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c index e6090a9dad..6bd3bb7700 100644 --- a/src/backend/access/rmgrdesc/xlogdesc.c +++ b/src/backend/access/rmgrdesc/xlogdesc.c @@ -112,12 +112,14 @@ xlog_desc(StringInfo buf, XLogReaderState *record) appendStringInfo(buf, "max_connections=%d max_worker_processes=%d " "max_wal_senders=%d max_prepared_xacts=%d " + "max_prepared_foreign_transactions=%d" "max_locks_per_xact=%d wal_level=%s " - "wal_log_hints=%s track_commit_timestamp=%s", + "wal_log_hints=%s track_commit_timestamp=%s ", xlrec.MaxConnections, xlrec.max_worker_processes, xlrec.max_wal_senders, xlrec.max_prepared_xacts, + xlrec.max_prepared_foreign_xacts, xlrec.max_locks_per_xact, wal_level_str, xlrec.wal_log_hints ? "on" : "off", diff --git a/src/backend/access/transam/Makefile b/src/backend/access/transam/Makefile index b05a88549d..26a5ee589c 100644 --- a/src/backend/access/transam/Makefile +++ b/src/backend/access/transam/Makefile @@ -16,6 +16,8 @@ OBJS = \ clog.o \ commit_ts.o \ fdwxact.o \ + fdwxact_launcher.o \ + fdwxact_resolver.o \ generic_xlog.o \ multixact.o \ parallel.o \ diff --git a/src/backend/access/transam/fdwxact.c b/src/backend/access/transam/fdwxact.c index ae3fdbdf83..0c6e80a6de 100644 --- a/src/backend/access/transam/fdwxact.c +++ b/src/backend/access/transam/fdwxact.c @@ -13,6 +13,57 @@ * transaction manager calls corresponding FDW API to end the foreign * tranasctions. * + * To achieve commit among all foreign servers atomically, the global transaction + * manager supports two-phase commit protocol, which is a type of atomic commitment + * protocol. We WAL log the foreign transaction state so foreign transaction state + * is crash-safe. + * + * FOREIGN TRANSACTION RESOLUTION + * + * At PREPARE TRANSACTION, we prepare all transactions on foreign servers by executing + * PrepareForeignTransaction() API for each foreign transaction regardless of data on + * the foreign server having been modified. At COMMIT PREPARED and ROLLBACK PREPARED, + * we commit or rollback only the local transaction but not do anything for involved + * foreign transactions. The prepared foreign transactinos are resolved by a resolver + * process asynchronously. Also, users can use pg_resolve_foreign_xact() SQL function + * that resolve a foreign transaction manually. + * + * LOCKING + * + * Whenever a foreign transaction is processed, the corresponding FdwXactState + * entry is updated. To avoid holding the lock during transaction processing + * which may take an unpredictable time the in-memory data of foreign + * transaction follows a locking model based on the following linked concepts: + * + * * A process who is going to work on the foreign transaction needs to set + * locking_backend of the FdwXactState entry, which prevents the entry from being + * updated and removed by concurrent processes. + * * All FdwXactState fields except for status are protected by FdwXactLock. The + * status is protected by its mutex. + * + * RECOVERY + * + * During replay WAL and replication FdwXactCtl also holds information about + * active prepared foreign transaction that haven't been moved to disk yet. + * + * Replay of fdwxact records happens by the following rules: + * + * * At the beginning of recovery, pg_fdwxacts is scanned once, filling FdwXactState + * with entries marked with fdwxact->inredo and fdwxact->ondisk. FdwXactState file + * data older than the XID horizon of the redo position are discarded. + * * On PREPARE redo, the foreign transaction is added to FdwXactCtl->xacts. + * We set fdwxact->inredo to true for such entries. + * * On Checkpoint we iterate through FdwXactCtl->xacts entries that + * have fdwxact->inredo set and are behind the redo_horizon. We save + * them to disk and then set fdwxact->ondisk to true. + * * On resolution we delete the entry from FdwXactCtl->xacts. If + * fdwxact->ondisk is true, the corresponding entry from the disk is + * additionally deleted. + * * RecoverFdwXacts() and PrescanFdwXacts() have been modified to go through + * fdwxact->inredo entries that have not made it to disk. + * + * These replay rules are borrowed from twophase.c + * * Portions Copyright (c) 2021, PostgreSQL Global Development Group * * IDENTIFICATION @@ -21,15 +72,42 @@ */ #include "postgres.h" +#include +#include +#include + #include "access/fdwxact.h" +#include "access/fdwxact_resolver.h" +#include "access/fdwxact_launcher.h" #include "access/xact.h" #include "access/xlog.h" +#include "access/twophase.h" +#include "access/resolver_internal.h" +#include "access/xact.h" +#include "access/xlog.h" +#include "access/xloginsert.h" +#include "access/xlogutils.h" #include "catalog/pg_user_mapping.h" #include "foreign/fdwapi.h" #include "foreign/foreign.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "pgstat.h" +#include "replication/syncrep.h" +#include "storage/ipc.h" +#include "storage/latch.h" +#include "storage/lock.h" +#include "storage/procarray.h" +#include "storage/sinvaladt.h" +#include "utils/builtins.h" +#include "utils/guc.h" #include "utils/memutils.h" +#include "utils/rel.h" #include "utils/syscache.h" +/* Directory where the foreign prepared transaction files will reside */ +#define FDWXACTS_DIR "pg_fdwxact" + /* Initial size of the hash table */ #define FDWXACT_HASH_SIZE 64 @@ -37,6 +115,23 @@ #define ServerSupportTransactionCallback(fdwent) \ (((FdwXactEntry *)(fdwent))->commit_foreign_xact_fn != NULL) +/* Check the FdwXactEntry is capable of two-phase commit */ +#define ServerSupportTwophaseCommit(fdwent) \ + (((FdwXactEntry *)(fdwent))->prepare_foreign_xact_fn != NULL) + +/* + * Name of foreign prepared transaction file is 8 bytes xid and + * user mapping OID separated by '_'. + * + * Since FdwXactState is identified by user mapping OID and it's unique + * within a distributed transaction, the name is fairly enough to + * ensure uniqueness. + */ +#define FDWXACT_FILE_NAME_LEN (8 + 1 + 8) +#define FdwXactStateFilePath(path, xid, umid) \ + snprintf(path, MAXPGPATH, FDWXACTS_DIR "/%08X_%08X", \ + xid, umid) + /* * Structure to bundle the foreign transaction participant. * @@ -51,26 +146,146 @@ typedef struct FdwXactEntry ForeignServer *server; UserMapping *usermapping; + /* + * Pointer to a FdwXactState entry in the global array. NULL if the entry + * is not inserted yet but this is registered as a participant. + */ + FdwXactState fdwxact; + /* Callbacks for foreign transaction */ CommitForeignTransaction_function commit_foreign_xact_fn; RollbackForeignTransaction_function rollback_foreign_xact_fn; + PrepareForeignTransaction_function prepare_foreign_xact_fn; } FdwXactEntry; /* - * Foreign transaction participants involved in the current transaction. - * A member of participants must support both commit and rollback APIs + * The current distributed transaction state. Members of participants + * must support at least both commit and rollback APIs * (i.g., ServerSupportTransactionCallback() is true). */ -static HTAB *FdwXactParticipants = NULL; +typedef struct DistributedXactStateData +{ + bool local_prepared; /* will (did) we prepare the local transaction? */ + + /* Statistics of participants */ + int nparticipants_no_twophase; /* how many participants doesn't + * support two-phase commit + * protocol? */ + + HTAB *participants; /* foreign transaction participants (FdwXactEntry) */ + List *serveroids_uniq; /* list of unique server OIDs in + * participants */ +} DistributedXactStateData; +static DistributedXactStateData DistributedXactState = { + .local_prepared = false, + .nparticipants_no_twophase = 0, + .participants = NULL, + .serveroids_uniq = NIL, +}; /* Check the current transaction has at least one fdwxact participant */ #define HasFdwXactParticipant() \ - (FdwXactParticipants != NULL && \ - hash_get_num_entries(FdwXactParticipants) > 0) + (DistributedXactState.participants != NULL && \ + hash_get_num_entries(DistributedXactState.participants) > 0) + +/* Keep track of registering process exit call back. */ +static bool fdwXactExitRegistered = false; + +/* Guc parameter */ +int max_prepared_foreign_xacts = 0; +int max_foreign_xact_resolvers = 0; +static void RemoveFdwXactEntry(Oid umid); static void EndFdwXactEntry(FdwXactEntry *fdwent, bool isCommit, bool is_parallel_worker); -static void RemoveFdwXactEntry(Oid umid); +static char *getFdwXactIdentifier(FdwXactEntry *fdwent, TransactionId xid); +static void ForgetAllParticipants(void); +static void FdwXactLaunchResolvers(void); + +static void PrepareAllFdwXacts(TransactionId xid); +static XLogRecPtr FdwXactInsertEntry(TransactionId xid, FdwXactEntry *fdwent, + char *identifier); +static void AtProcExit_FdwXact(int code, Datum arg); +static void FdwXactComputeRequiredXmin(void); +static FdwXactStatus FdwXactGetTransactionFate(TransactionId xid); +static void FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn); +static void FdwXactRedoRemove(TransactionId xid, Oid umid, bool givewarning); +static void XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len); +static char *ProcessFdwXactBuffer(TransactionId xid, Oid umid, + XLogRecPtr insert_start_lsn, bool fromdisk); +static char *ReadFdwXactStateFile(TransactionId xid, Oid umid); +static void RemoveFdwXactStateFile(TransactionId xid, Oid umid, bool giveWarning); +static void RecreateFdwXactFile(TransactionId xid, Oid umid, void *content, int len); + +static FdwXactState insert_fdwxact(Oid dbid, TransactionId xid, Oid umid, Oid serverid, + Oid owner, char *identifier); +static void remove_fdwxact(FdwXactState fdwxact); +static List *find_fdwxacts(TransactionId xid, Oid umid, Oid dbid); +static FdwXactState get_fdwxact_with_check(TransactionId xid, Oid umid, + bool check_two_phase); +static void pg_foreign_xact_callback(int code, Datum arg); + +/* + * Calculates the size of shared memory allocated for maintaining foreign + * prepared transaction entries. + */ +Size +FdwXactShmemSize(void) +{ + Size size; + + /* Size for foreign transaction information array */ + size = offsetof(FdwXactCtlData, xacts); + size = add_size(size, mul_size(max_prepared_foreign_xacts, + sizeof(FdwXactState))); + size = MAXALIGN(size); + size = add_size(size, mul_size(max_prepared_foreign_xacts, + sizeof(FdwXactStateData))); + + return size; +} + +/* + * Initialization of shared memory for maintaining foreign prepared transaction + * entries. The shared memory layout is defined in definition of FdwXactCtlData + * structure. + */ +void +FdwXactShmemInit(void) +{ + bool found; + + FdwXactCtl = ShmemInitStruct("Foreign transactions table", + FdwXactShmemSize(), + &found); + if (!IsUnderPostmaster) + { + FdwXactState fdwxacts; + int cnt; + + Assert(!found); + FdwXactCtl->free_fdwxacts = NULL; + FdwXactCtl->num_xacts = 0; + + /* Initialize the linked list of free FDW transactions */ + fdwxacts = (FdwXactState) + ((char *) FdwXactCtl + + MAXALIGN(offsetof(FdwXactCtlData, xacts) + + sizeof(FdwXactState) * max_prepared_foreign_xacts)); + for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++) + { + fdwxacts[cnt].status = FDWXACT_STATUS_INVALID; + fdwxacts[cnt].fdwxact_free_next = FdwXactCtl->free_fdwxacts; + FdwXactCtl->free_fdwxacts = &fdwxacts[cnt]; + SpinLockInit(&fdwxacts[cnt].mutex); + } + } + else + { + Assert(FdwXactCtl); + Assert(found); + } +} /* * Register the given foreign transaction participant identified by the @@ -87,20 +302,21 @@ FdwXactRegisterEntry(UserMapping *usermapping) Assert(IsTransactionState()); - if (FdwXactParticipants == NULL) + if (DistributedXactState.participants == NULL) { HASHCTL ctl; ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(FdwXactEntry); - FdwXactParticipants = hash_create("fdw xact participants", - FDWXACT_HASH_SIZE, - &ctl, HASH_ELEM | HASH_BLOBS); + DistributedXactState.participants = hash_create("fdw xact participants", + FDWXACT_HASH_SIZE, + &ctl, HASH_ELEM | HASH_BLOBS); } umid = usermapping->umid; - fdwent = hash_search(FdwXactParticipants, (void *) &umid, HASH_ENTER, &found); + fdwent = hash_search(DistributedXactState.participants, + (void *) &umid, HASH_ENTER, &found); if (found) return; @@ -124,13 +340,22 @@ FdwXactRegisterEntry(UserMapping *usermapping) ereport(ERROR, (errmsg("cannot register foreign server not supporting transaction callback"))); + fdwent->fdwxact = NULL; fdwent->commit_foreign_xact_fn = routine->CommitForeignTransaction; fdwent->rollback_foreign_xact_fn = routine->RollbackForeignTransaction; + fdwent->prepare_foreign_xact_fn = routine->PrepareForeignTransaction; MemoryContextSwitchTo(old_ctx); + + /* Update statistics */ + if (!ServerSupportTwophaseCommit(fdwent)) + DistributedXactState.nparticipants_no_twophase++; + + Assert(DistributedXactState.nparticipants_no_twophase <= + hash_get_num_entries(DistributedXactState.participants)); } -/* Remove the foreign transaction from FdwXactParticipants */ +/* Remove the foreign transaction from the current participants */ void FdwXactUnregisterEntry(UserMapping *usermapping) { @@ -145,7 +370,21 @@ FdwXactUnregisterEntry(UserMapping *usermapping) static void RemoveFdwXactEntry(Oid umid) { - (void) hash_search(FdwXactParticipants, (void *) &umid, HASH_REMOVE, NULL); + FdwXactEntry *fdwent; + + Assert(DistributedXactState.participants != NULL); + fdwent = hash_search(DistributedXactState.participants, (void *) &umid, + HASH_REMOVE, NULL); + + if (fdwent) + { + /* Update statistics */ + if (!ServerSupportTwophaseCommit(fdwent)) + DistributedXactState.nparticipants_no_twophase--; + + Assert(DistributedXactState.nparticipants_no_twophase <= + hash_get_num_entries(DistributedXactState.participants)); + } } /* @@ -154,29 +393,75 @@ RemoveFdwXactEntry(Oid umid) void AtEOXact_FdwXact(bool isCommit, bool is_parallel_worker) { - FdwXactEntry *fdwent; - HASH_SEQ_STATUS scan; - /* If there are no foreign servers involved, we have no business here */ if (!HasFdwXactParticipant()) return; - hash_seq_init(&scan, FdwXactParticipants); - while ((fdwent = (FdwXactEntry *) hash_seq_search(&scan))) + Assert(!RecoveryInProgress()); + + if (!isCommit) { - Assert(ServerSupportTransactionCallback(fdwent)); + HASH_SEQ_STATUS scan; + FdwXactEntry *fdwent; - /* Commit or rollback foreign transaction */ - EndFdwXactEntry(fdwent, isCommit, is_parallel_worker); + /* Rollback foreign transactions in the participant list */ + hash_seq_init(&scan, DistributedXactState.participants); + while ((fdwent = (FdwXactEntry *) hash_seq_search(&scan))) + { + FdwXactState fdwxact = fdwent->fdwxact; + int status; - /* - * Remove the entry so that we don't recursively process this foreign - * transaction. - */ - RemoveFdwXactEntry(fdwent->umid); + /* + * If this foreign transaction is not prepared yet, end the + * foreign transaction in one-phase. + */ + if (!fdwxact) + { + Assert(ServerSupportTransactionCallback(fdwent)); + EndFdwXactEntry(fdwent, false, is_parallel_worker); + + /* + * Remove FdwXactState entry to prevent processing again in a + * recursive error case. + */ + RemoveFdwXactEntry(fdwent->umid); + continue; + } + + /* + * If the foreign transaction has FdwXactState entry, the foreign + * transaction might have been prepared. We rollback the foreign + * transaction anyway to end the current transaction if the status + * is in-progress. Since the transaction might have been already + * prepared on the foreign we set the status to aborting and leave + * it. + */ + SpinLockAcquire(&(fdwxact->mutex)); + status = fdwxact->status; + fdwxact->status = FDWXACT_STATUS_ABORTING; + SpinLockRelease(&(fdwxact->mutex)); + + if (status == FDWXACT_STATUS_PREPARING) + EndFdwXactEntry(fdwent, isCommit, is_parallel_worker); + } } - Assert(!HasFdwXactParticipant()); + /* Unlock all participants */ + ForgetAllParticipants(); + + /* + * Launch the resolver processes if we failed to prepare the local + * transaction after preparing the foreign transactions. In this case, we + * need to rollback the prepared transaction on the foreign servers. + */ + if (DistributedXactState.local_prepared && !isCommit) + FdwXactLaunchResolvers(); + + /* Reset all fields */ + DistributedXactState.local_prepared = false; + DistributedXactState.nparticipants_no_twophase = 0; + list_free(DistributedXactState.serveroids_uniq); + DistributedXactState.serveroids_uniq = NIL; } /* @@ -193,6 +478,7 @@ EndFdwXactEntry(FdwXactEntry *fdwent, bool isCommit, bool is_parallel_worker) finfo.usermapping = fdwent->usermapping; finfo.flags = FDWXACT_FLAG_ONEPHASE | ((is_parallel_worker) ? FDWXACT_FLAG_PARALLEL_WORKER : 0); + finfo.identifier = NULL; if (isCommit) { @@ -209,15 +495,1610 @@ EndFdwXactEntry(FdwXactEntry *fdwent, bool isCommit, bool is_parallel_worker) } /* - * This function is called at PREPARE TRANSACTION. Since we don't support - * preparing foreign transactions for now, raise an error if the local transaction - * has any foreign transaction. + * Prepare foreign transactions by PREPARE TRANSACTION command. + * + * In case where an error happens during parparing a foreign transaction we + * change to rollback. See AtEOXact_FdwXact() for details. */ void AtPrepare_FdwXact(void) { - if (HasFdwXactParticipant()) + TransactionId xid; + + /* If there are no foreign servers involved, we have no business here */ + if (!HasFdwXactParticipant()) + return; + + /* + * Check if there is a server that doesn't support two-phase commit. All + * involved servers need to support two-phase commit as we're going to + * prepare all of them. + */ + if (DistributedXactState.nparticipants_no_twophase > 0) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot PREPARE a transaction that has operated on foreign tables"))); + errmsg("cannot PREPARE a distributed transaction that has operated on a foreign server not supporting two-phase commit protocol"))); + + /* + * Assign a transaction id if not yet because the local transaction id + * is needed to determine the result of the distributed transaction. + */ + xid = GetTopTransactionId(); + + /* + * Mark the local transaction will be prepared before actually preparing + * any foreign trasactions so that in a case where an error happens during + * preparing a foreign transaction or preparing the local transaction, we can + * launch a resolver to rollback already-prepared foreign transactions. + */ + DistributedXactState.local_prepared = true; + + PrepareAllFdwXacts(xid); +} + +/* + * Pre-commit processing for foreign transactions. We commit those foreign + * transactions with one-phase. + */ +void +PreCommit_FdwXact(bool is_parallel_worker) +{ + HASH_SEQ_STATUS scan; + FdwXactEntry *fdwent; + + /* + * If there is no foreign server involved or all foreign transactions are + * already prepared (see AtPrepare_FdwXact()), we have no business here. + */ + if (!HasFdwXactParticipant() || DistributedXactState.local_prepared) + return; + + Assert(!RecoveryInProgress()); + + /* Commit all foreign transactions in the participant list */ + hash_seq_init(&scan, DistributedXactState.participants); + while ((fdwent = (FdwXactEntry *) hash_seq_search(&scan))) + { + Assert(ServerSupportTransactionCallback(fdwent)); + + /* + * Commit the foreign transaction and remove itself from the hash + * table so that we don't try to abort already-closed transaction. + */ + EndFdwXactEntry(fdwent, true, is_parallel_worker); + RemoveFdwXactEntry(fdwent->umid); + } +} + +/* + * Functions to count the number of FdwXactState entries associated with + * the given argument. + */ +int +CountFdwXactsForUserMapping(Oid umid) +{ + List *res; + + LWLockAcquire(FdwXactLock, LW_SHARED); + res = find_fdwxacts(InvalidTransactionId, umid, InvalidOid); + LWLockRelease(FdwXactLock); + + return list_length(res); +} + +int +CountFdwXactsForDB(Oid dbid) +{ + List *res; + + LWLockAcquire(FdwXactLock, LW_SHARED); + res = find_fdwxacts(InvalidTransactionId, InvalidOid, dbid); + LWLockRelease(FdwXactLock); + + return list_length(res); + +} + +/* + * We must fsync the foreign transaction state file that is valid or generated + * during redo and has a inserted LSN <= the checkpoint's redo horizon. + * The foreign transaction entries and hence the corresponding files are expected + * to be very short-lived. By executing this function at the end, we might have + * lesser files to fsync, thus reducing some I/O. This is similar to + * CheckPointTwoPhase(). + * + * This is deliberately run as late as possible in the checkpoint sequence, + * because FdwXacts ordinarily have short lifespans, and so it is quite + * possible that FdwXactStates that were valid at checkpoint start will no longer + * exist if we wait a little bit. With typical checkpoint settings this + * will be about 3 minutes for an online checkpoint, so as a result we + * expect that there will be no FdwXactStates that need to be copied to disk. + * + * If a FdwXactState remains valid across multiple checkpoints, it will already + * be on disk so we don't bother to repeat that write. + */ +void +CheckPointFdwXacts(XLogRecPtr redo_horizon) +{ + int cnt; + int serialized_fdwxacts = 0; + + if (max_prepared_foreign_xacts == 0) + return; /* nothing to do */ + + /* + * We are expecting there to be zero FdwXactState that need to be copied + * to disk, so we perform all I/O while holding FdwXactLock for + * simplicity. This presents any new foreign xacts from preparing while + * this occurs, which shouldn't be a problem since the presence of + * long-lived prepared foreign xacts indicated the transaction manager + * isn't active. + * + * It's also possible to move I/O out of the lock, but on every error we + * should check whether somebody committed our transaction in different + * backend. Let's leave this optimisation for future, if somebody will + * spot that this place cause bottleneck. + * + * Note that it isn't possible for there to be a FdwXactState with a + * insert_end_lsn set prior to the last checkpoint yet is marked invalid, + * because of the efforts with delayChkpt. + */ + LWLockAcquire(FdwXactLock, LW_SHARED); + for (cnt = 0; cnt < FdwXactCtl->num_xacts; cnt++) + { + FdwXactState fdwxact = FdwXactCtl->xacts[cnt]; + + if ((fdwxact->valid || fdwxact->inredo) && + !fdwxact->ondisk && + fdwxact->insert_end_lsn <= redo_horizon) + { + char *buf; + int len; + + XlogReadFdwXactData(fdwxact->insert_start_lsn, &buf, &len); + RecreateFdwXactFile(fdwxact->data.xid, fdwxact->data.umid, buf, len); + fdwxact->ondisk = true; + fdwxact->insert_start_lsn = InvalidXLogRecPtr; + fdwxact->insert_end_lsn = InvalidXLogRecPtr; + pfree(buf); + serialized_fdwxacts++; + } + } + + LWLockRelease(FdwXactLock); + + /* + * Flush unconditionally the parent directory to make any information + * durable on disk. FdwXactState files could have been removed and those + * removals need to be made persistent as well as any files newly created. + */ + fsync_fname(FDWXACTS_DIR, true); + + if (log_checkpoints && serialized_fdwxacts > 0) + ereport(LOG, + (errmsg_plural("%u foreign transaction state file was written " + "for long-running prepared transactions", + "%u foreign transaction state files were written " + "for long-running prepared transactions", + serialized_fdwxacts, + serialized_fdwxacts))); +} + +/* + * Prepare all foreign transactions. + * + * The basic strategy is to create all FdwXactState entries with WAL-logging, + * wait for those WAL records to be replicated if synchronous replication is + * enabled, and then prepare foreign transactions by calling + * PrepareForeignTransaction FDW callback functions. There are two points + * here: (a) writing WAL records before preparing foreign transactions, and + * (b) waiting for those records to replicated before preparing foreign + * transactions. + * + * You might think that we can itereate over foreign transaction while preparing + * the foreign transaction and writing WAL record one by one. But this doesn't + * work if the server crashes in the middle of those operations. We will end + * up losing foreign prepared transaction information if the server crashes + * and/or the failover happens. Therefore, we need point (a) because otherwise + * we will lost the prepared transaction on the foreign server and will not be + * able to resolve it after the server crash. Hence persist first then prepare. + * Point (b) guarantees that foreign transaction information are not lost even + * if the failover happens. + */ +static void +PrepareAllFdwXacts(TransactionId xid) +{ + FdwXactEntry *fdwent; + XLogRecPtr flush_lsn; + bool canceled; + HASH_SEQ_STATUS scan; + + Assert(TransactionIdIsValid(xid)); + + /* Persist all foreign transaction entries */ + hash_seq_init(&scan, DistributedXactState.participants); + while ((fdwent = (FdwXactEntry *) hash_seq_search(&scan))) + { + char *identifier; + + Assert(ServerSupportTwophaseCommit(fdwent)); + + CHECK_FOR_INTERRUPTS(); + + /* Get prepared transaction identifier */ + identifier = getFdwXactIdentifier(fdwent, xid); + Assert(identifier); + + /* + * Insert the foreign transaction entry with the + * FDWXACT_STATUS_PREPARING status. Registration persists this + * information to the disk and logs (that way relaying it on standby). + * Thus in case we loose connectivity to the foreign server or crash + * ourselves, we will remember that we might have prepared transaction + * on the foreign server and try to resolve it when connectivity is + * restored or after crash recovery. + */ + flush_lsn = FdwXactInsertEntry(xid, fdwent, identifier); + } + + HOLD_INTERRUPTS(); + + /* Wait for all WAL records to be replicated, if necessary */ + canceled = SyncRepWaitForLSN(flush_lsn, false); + + RESUME_INTERRUPTS(); + + /* + * XXX: dirty hack to invoke an interruption that was absorbed. + * SyncRepWaitForLSN() is aimed to be used at the point where we never be + * able to change to rollback. But at this state, preparing foreign + * transactions at pre-commit phase, we still are able to change to rollback. + * So if the waits canceled by an interruption, we error out. Note that + * SyncRepWaitForLSN() might have been set ProcDiePending too. + */ + if (canceled) + { + QueryCancelPending = true; + ProcessInterrupts(); + } + + /* Prepare all foreign transactions */ + hash_seq_init(&scan, DistributedXactState.participants); + while ((fdwent = (FdwXactEntry *) hash_seq_search(&scan))) + { + FdwXactInfo finfo; + FdwXactState fdwxact = fdwent->fdwxact; + + /* + * Prepare the foreign transaction. Between FdwXactInsertEntry call + * till this backend hears acknowledge from foreign server, the + * backend may abort the local transaction (say, because of a signal). + */ + finfo.server = fdwent->server; + finfo.usermapping = fdwent->usermapping; + finfo.flags = 0; + finfo.identifier = pstrdup(fdwxact->data.identifier); + fdwent->prepare_foreign_xact_fn(&finfo); + + /* succeeded, update status */ + SpinLockAcquire(&fdwxact->mutex); + fdwent->fdwxact->status = FDWXACT_STATUS_PREPARED; + SpinLockRelease(&fdwxact->mutex); + + pfree(finfo.identifier); + } +} + +/* + * Return a null-terminated foreign transaction identifier. We generate an + * unique identifier with in the form of + * "fx___ whose length is less than FDWXACT_ID_MAX_LEN. + * + * Returned string value is used to identify foreign transaction. The + * identifier should not be same as any other concurrent prepared transaction + * identifier. + * + * To make the foreign transactionid unique, we should ideally use something + * like UUID, which gives unique ids with high probability, but that may be + * expensive here and UUID extension which provides the function to generate + * UUID is not part of the core code. + */ +static char * +getFdwXactIdentifier(FdwXactEntry *fdwent, TransactionId xid) +{ + char buf[FDWXACT_ID_MAX_LEN] = {0}; + + snprintf(buf, FDWXACT_ID_MAX_LEN, "fx_%ld_%u_%u", Abs(random()), + xid, fdwent->umid); + + return pstrdup(buf); +} + +/* + * This function insert a new FdwXactState entry to the global array with + * WAL-logging. The new entry is held by the backend who inserted. + */ +static XLogRecPtr +FdwXactInsertEntry(TransactionId xid, FdwXactEntry *fdwent, char *identifier) +{ + FdwXactStateOnDiskData *fdwxact_file_data; + FdwXactState fdwxact; + Oid owner; + int data_len; + + /* on first call, register the exit hook */ + if (!fdwXactExitRegistered) + { + before_shmem_exit(AtProcExit_FdwXact, 0); + fdwXactExitRegistered = true; + } + + /* + * Enter the foreign transaction into the shared memory structure. + */ + owner = GetUserId(); + LWLockAcquire(FdwXactLock, LW_EXCLUSIVE); + fdwxact = insert_fdwxact(MyDatabaseId, xid, fdwent->umid, + fdwent->usermapping->serverid, owner, identifier); + fdwxact->locking_backend = MyBackendId; + LWLockRelease(FdwXactLock); + + /* Update FdwXactEntry */ + fdwent->fdwxact = fdwxact; + + /* Remember server's oid where we prepared the transaction */ + DistributedXactState.serveroids_uniq = + list_append_unique_oid(DistributedXactState.serveroids_uniq, + fdwent->usermapping->serverid); + + /* + * Prepare to write the entry to a file. Also add xlog entry. The contents + * of the xlog record are same as what is written to the file. + */ + data_len = offsetof(FdwXactStateOnDiskData, identifier); + data_len = data_len + strlen(identifier) + 1; + data_len = MAXALIGN(data_len); + fdwxact_file_data = (FdwXactStateOnDiskData *) palloc0(data_len); + memcpy(fdwxact_file_data, &(fdwxact->data), data_len); + + START_CRIT_SECTION(); + + /* See note in RecordTransactionCommit */ + MyProc->delayChkpt = true; + + /* Add the entry in the xlog and save LSN for checkpointer */ + XLogBeginInsert(); + XLogRegisterData((char *) fdwxact_file_data, data_len); + fdwxact->insert_end_lsn = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_INSERT); + + /* If we crash now, we have prepared: WAL replay will fix things */ + + /* Store record's start location to read that later on CheckPoint */ + fdwxact->insert_start_lsn = ProcLastRecPtr; + + /* File is written completely, checkpoint can proceed with syncing */ + fdwxact->valid = true; + + /* Checkpoint can process now */ + MyProc->delayChkpt = false; + + END_CRIT_SECTION(); + + pfree(fdwxact_file_data); + return fdwxact->insert_end_lsn; +} + +/* + * Insert a new entry for a given foreign transaction identified by transaction + * id, foreign server and user mapping, into the shared memory array. Caller + * must hold FdwXactLock in exclusive mode. + * + * If the entry already exists, the function raises an error. + */ +static FdwXactState +insert_fdwxact(Oid dbid, TransactionId xid, Oid umid, Oid serverid, Oid owner, + char *identifier) +{ + FdwXactState fdwxact; + + Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE)); + + /* Check for duplicated foreign transaction entry */ + for (int i = 0; i < FdwXactCtl->num_xacts; i++) + { + fdwxact = FdwXactCtl->xacts[i]; + if (fdwxact->valid && + fdwxact->data.xid == xid && + fdwxact->data.umid == umid) + ereport(ERROR, + (errmsg("could not insert a foreign transaction entry"), + errdetail("Duplicate entry with transaction id %u, user mapping id %u exists.", + xid, umid))); + } + + /* + * Get a next free foreign transaction entry. Raise error if there are + * none left. + */ + if (!FdwXactCtl->free_fdwxacts) + { + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("maximum number of foreign transactions reached"), + errhint("Increase max_prepared_foreign_transactions: \"%d\".", + max_prepared_foreign_xacts))); + } + fdwxact = FdwXactCtl->free_fdwxacts; + FdwXactCtl->free_fdwxacts = fdwxact->fdwxact_free_next; + + /* Insert the entry to shared memory array */ + Assert(FdwXactCtl->num_xacts < max_prepared_foreign_xacts); + FdwXactCtl->xacts[FdwXactCtl->num_xacts++] = fdwxact; + + fdwxact->status = FDWXACT_STATUS_PREPARING; + fdwxact->data.xid = xid; + fdwxact->data.dbid = dbid; + fdwxact->data.umid = umid; + fdwxact->data.serverid = serverid; + fdwxact->data.owner = owner; + strlcpy(fdwxact->data.identifier, identifier, FDWXACT_ID_MAX_LEN); + fdwxact->data.identifier[strlen(identifier)] = '\0'; + + fdwxact->insert_start_lsn = InvalidXLogRecPtr; + fdwxact->insert_end_lsn = InvalidXLogRecPtr; + fdwxact->locking_backend = InvalidBackendId; + fdwxact->valid = false; + fdwxact->ondisk = false; + fdwxact->inredo = false; + + return fdwxact; +} + +/* + * Remove the foreign prepared transaction entry from shared memory. + * Caller must hold FdwXactLock in exclusive mode. + */ +static void +remove_fdwxact(FdwXactState fdwxact) +{ + int i; + + Assert(fdwxact != NULL); + Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE)); + + elog(DEBUG2, "remove fdwxact entry id %s", fdwxact->data.identifier); + + if (!RecoveryInProgress()) + { + xl_fdwxact_remove record; + XLogRecPtr recptr; + + /* Fill up the log record before releasing the entry */ + record.xid = fdwxact->data.xid; + record.umid = fdwxact->data.umid; + + /* + * Now writing FdwXactState data to WAL. We have to set delayChkpt + * here, otherwise a checkpoint starting immediately after the WAL + * record is inserted could complete without fsync'ing our state file. + * (This is essentially the same kind of race condition as the + * COMMIT-to-clog-write case that RecordTransactionCommit uses + * delayChkpt for; see notes there.) + */ + START_CRIT_SECTION(); + + MyProc->delayChkpt = true; + + /* + * Log that we are removing the foreign transaction entry and remove + * the file from the disk as well. + */ + XLogBeginInsert(); + XLogRegisterData((char *) &record, sizeof(xl_fdwxact_remove)); + recptr = XLogInsert(RM_FDWXACT_ID, XLOG_FDWXACT_REMOVE); + + /* Always flush, since we're about to remove the FdwXact state file */ + XLogFlush(recptr); + + /* Now we can mark ourselves as out of the commit critical section */ + MyProc->delayChkpt = false; + + END_CRIT_SECTION(); + } + + /* Search the slot where this entry resided */ + for (i = 0; i < FdwXactCtl->num_xacts; i++) + { + if (FdwXactCtl->xacts[i] == fdwxact) + break; + } + + Assert(i < FdwXactCtl->num_xacts); + + /* Clean up any files we may have left */ + if (fdwxact->ondisk) + RemoveFdwXactStateFile(fdwxact->data.xid, fdwxact->data.umid, true); + + /* Remove the entry from active array */ + FdwXactCtl->num_xacts--; + FdwXactCtl->xacts[i] = FdwXactCtl->xacts[FdwXactCtl->num_xacts]; + + /* Put it back into free list */ + fdwxact->fdwxact_free_next = FdwXactCtl->free_fdwxacts; + FdwXactCtl->free_fdwxacts = fdwxact; + + /* Reset informations */ + fdwxact->status = FDWXACT_STATUS_INVALID; + fdwxact->locking_backend = InvalidBackendId; + fdwxact->valid = false; + fdwxact->ondisk = false; + fdwxact->inredo = false; +} + +/* + * When the process exits, unlock all the entries. + */ +static void +AtProcExit_FdwXact(int code, Datum arg) +{ + ForgetAllParticipants(); + FdwXactLaunchResolvers(); +} + +/* + * Unlock all foreign transaction participants. If we left foreign transaction, + * update the oldest xmin of unresolved transaction to prevent the local + * transaction id of such unresolved foreign transaction from begin truncated. + * Returns the number of remaining foreign transactions. We return the unique + * list of foreign servers' oids so that the caller request to launch the resolver + * using it. + */ +static void +ForgetAllParticipants(void) +{ + FdwXactEntry *fdwent; + HASH_SEQ_STATUS scan; + int nremaining = 0; + + if (!HasFdwXactParticipant()) + return; + + hash_seq_init(&scan, DistributedXactState.participants); + while ((fdwent = (FdwXactEntry *) hash_seq_search(&scan))) + { + FdwXactState fdwxact = fdwent->fdwxact; + + if (fdwxact) + { + Assert(fdwxact->locking_backend == MyBackendId); + + /* Unlock the foreign transaction entry */ + LWLockAcquire(FdwXactLock, LW_EXCLUSIVE); + fdwxact->locking_backend = InvalidBackendId; + LWLockRelease(FdwXactLock); + + nremaining++; + } + + /* Remove from the participants list */ + RemoveFdwXactEntry(fdwent->umid); + } + + /* + * If we leave any FdwXactState entries, update the oldest local + * transaction of unresolved distributed transaction. + */ + if (nremaining > 0) + FdwXactComputeRequiredXmin(); + + Assert(!HasFdwXactParticipant()); +} + +static void +FdwXactLaunchResolvers(void) +{ + if (list_length(DistributedXactState.serveroids_uniq) > 0) + LaunchOrWakeupFdwXactResolver(DistributedXactState.serveroids_uniq); +} + +/* + * Launch or wake up the resolver to resolve the given transaction. + */ +void +FdwXactLaunchResolversForXid(TransactionId xid) +{ + List *serveroids = NIL; + + if (max_prepared_foreign_xacts == 0) + return; /* nothing to do */ + + LWLockAcquire(FdwXactLock, LW_SHARED); + for (int i = 0; i < FdwXactCtl->num_xacts; i++) + { + FdwXactState fdwxact = FdwXactCtl->xacts[i]; + + if (!fdwxact->valid) + continue; + + /* Collect server oids associated with the given transaction */ + if (fdwxact->data.xid == xid) + serveroids = list_append_unique_oid(serveroids, fdwxact->data.serverid); + } + LWLockRelease(FdwXactLock); + + /* Exit if there is no servers to launch */ + if (serveroids == NIL) + return; + + LaunchOrWakeupFdwXactResolver(serveroids); + list_free(serveroids); +} + +/* + * Commit or rollback one prepared foreign transaction, and remove FdwXactState + * entry. + */ +void +ResolveOneFdwXact(FdwXactState fdwxact) +{ + FdwXactInfo finfo; + FdwRoutine *routine; + + /* The FdwXactState entry must be held by me */ + Assert(fdwxact != NULL); + Assert(fdwxact->locking_backend == MyBackendId); + Assert(fdwxact->status == FDWXACT_STATUS_PREPARED || + fdwxact->status == FDWXACT_STATUS_COMMITTING || + fdwxact->status == FDWXACT_STATUS_ABORTING); + + /* Set whether we do commit or abort if not set yet */ + if (fdwxact->status == FDWXACT_STATUS_PREPARED) + { + FdwXactStatus new_status; + + new_status = FdwXactGetTransactionFate(fdwxact->data.xid); + Assert(new_status == FDWXACT_STATUS_COMMITTING || + new_status == FDWXACT_STATUS_ABORTING); + + /* Update the status */ + SpinLockAcquire(&fdwxact->mutex); + fdwxact->status = new_status; + SpinLockRelease(&fdwxact->mutex); + } + + routine = GetFdwRoutineByServerId(fdwxact->data.serverid); + + /* Prepare the foreign transaction information to pass to API */ + finfo.server = GetForeignServer(fdwxact->data.serverid); + finfo.usermapping = GetUserMapping(fdwxact->data.owner, fdwxact->data.serverid); + finfo.flags = 0; + finfo.identifier = fdwxact->data.identifier; + + if (fdwxact->status == FDWXACT_STATUS_COMMITTING) + { + routine->CommitForeignTransaction(&finfo); + elog(DEBUG1, "successfully committed the prepared foreign transaction %s", + fdwxact->data.identifier); + } + else + { + routine->RollbackForeignTransaction(&finfo); + elog(DEBUG1, "successfully rolled back the prepared foreign transaction %s", + fdwxact->data.identifier); + } + + LWLockAcquire(FdwXactLock, LW_EXCLUSIVE); + remove_fdwxact(fdwxact); + LWLockRelease(FdwXactLock); +} + +/* + * Compute the oldest xmin across all unresolved foreign transactions + * and store it in the ProcArray. + */ +static void +FdwXactComputeRequiredXmin(void) +{ + TransactionId agg_xmin = InvalidTransactionId; + + Assert(FdwXactCtl != NULL); + + LWLockAcquire(FdwXactLock, LW_SHARED); + + for (int i = 0; i < FdwXactCtl->num_xacts; i++) + { + FdwXactState fdwxact = FdwXactCtl->xacts[i]; + + if (!fdwxact->valid) + continue; + + Assert(TransactionIdIsValid(fdwxact->data.xid)); + + /* + * We can exclude entries that are marked as either committing or + * aborting and its state file is on disk since such entries no longer + * need to lookup its transaction status from the commit log. + */ + if (!TransactionIdIsValid(agg_xmin) || + TransactionIdPrecedes(fdwxact->data.xid, agg_xmin) || + (fdwxact->ondisk && + (fdwxact->status == FDWXACT_STATUS_COMMITTING || + fdwxact->status == FDWXACT_STATUS_ABORTING))) + agg_xmin = fdwxact->data.xid; + } + + LWLockRelease(FdwXactLock); + + ProcArraySetFdwXactUnresolvedXmin(agg_xmin); +} + + +/* + * Return whether the foreign transaction associated with the given transaction + * id should be committed or rolled back according to the result of the local + * transaction. + */ +static FdwXactStatus +FdwXactGetTransactionFate(TransactionId xid) +{ + /* + * If the local transaction is already committed, commit prepared foreign + * transaction. + */ + if (TransactionIdDidCommit(xid)) + return FDWXACT_STATUS_COMMITTING; + + /* + * If the local transaction is already aborted, abort prepared foreign + * transactions. + */ + else if (TransactionIdDidAbort(xid)) + return FDWXACT_STATUS_ABORTING; + + /* + * The local transaction is not in progress but the foreign transaction is + * not prepared on the foreign server. This can happen when transaction + * failed after registered this entry but before actual preparing on the + * foreign server. So let's assume it aborted. + */ + else if (!TransactionIdIsInProgress(xid)) + return FDWXACT_STATUS_ABORTING; + + /* + * The Local transaction is in progress and foreign transaction is about + * to be committed or aborted. Raise an error anyway since we cannot + * determine the fate of this foreign transaction according to the local + * transaction whose fate is also not determined. + */ + elog(ERROR, + "cannot resolve the foreign transaction associated with in-process transaction"); + + pg_unreachable(); +} + + +/* + * Recreates a foreign transaction state file. This is used in WAL replay + * and during checkpoint creation. + * + * Note: content and len don't include CRC. + */ +void +RecreateFdwXactFile(TransactionId xid, Oid umid, void *content, int len) +{ + char path[MAXPGPATH]; + pg_crc32c statefile_crc; + int fd; + + /* Recompute CRC */ + INIT_CRC32C(statefile_crc); + COMP_CRC32C(statefile_crc, content, len); + FIN_CRC32C(statefile_crc); + + FdwXactStateFilePath(path, xid, umid); + + fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY); + + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not recreate foreign transaction state file \"%s\": %m", + path))); + + /* Write content and CRC */ + pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_WRITE); + if (write(fd, content, len) != len) + { + /* if write didn't set errno, assume problem is no disk space */ + if (errno == 0) + errno = ENOSPC; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write foreign transaction state file: %m"))); + } + if (write(fd, &statefile_crc, sizeof(pg_crc32c)) != sizeof(pg_crc32c)) + { + if (errno == 0) + errno = ENOSPC; + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write foreign transaction state file: %m"))); + } + pgstat_report_wait_end(); + + /* + * We must fsync the file because the end-of-replay checkpoint will not do + * so, there being no FDWXACT in shared memory yet to tell it to. + */ + pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_SYNC); + if (pg_fsync(fd) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not fsync foreign transaction state file: %m"))); + pgstat_report_wait_end(); + + if (CloseTransientFile(fd) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not close foreign transaction file: %m"))); +} + + +/* Apply the redo log for a foreign transaction */ +void +fdwxact_redo(XLogReaderState *record) +{ + char *rec = XLogRecGetData(record); + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + if (info == XLOG_FDWXACT_INSERT) + { + /* + * Add fdwxact entry and set start/end lsn of the WAL record in + * FdwXactState entry. + */ + LWLockAcquire(FdwXactLock, LW_EXCLUSIVE); + FdwXactRedoAdd(XLogRecGetData(record), record->ReadRecPtr, + record->EndRecPtr); + LWLockRelease(FdwXactLock); + } + else if (info == XLOG_FDWXACT_REMOVE) + { + xl_fdwxact_remove *record = (xl_fdwxact_remove *) rec; + + /* Delete FdwXactState entry and file if exists */ + LWLockAcquire(FdwXactLock, LW_EXCLUSIVE); + FdwXactRedoRemove(record->xid, record->umid, false); + LWLockRelease(FdwXactLock); + } + else + elog(ERROR, "invalid log type %d in foreign transaction log record", info); + + return; +} + +/* + * Scan the shared memory entries of FdwXactState and determine the range of valid + * XIDs present. This is run during database startup, after we have completed + * reading WAL. ShmemVariableCache->nextXid has been set to one more than + * the highest XID for which evidence exists in WAL. + + * On corrupted two-phase files, fail immediately. Keeping around broken + * entries and let replay continue causes harm on the system, and a new + * backup should be rolled in. + + * Our other responsibility is to update and return the oldest valid XID + * among the distributed transactions. This is needed to synchronize pg_subtrans + * startup properly. + */ +TransactionId +PrescanFdwXacts(TransactionId oldestActiveXid) +{ + FullTransactionId nextXid = ShmemVariableCache->nextXid; + TransactionId origNextXid = XidFromFullTransactionId(nextXid); + TransactionId result = origNextXid; + + LWLockAcquire(FdwXactLock, LW_EXCLUSIVE); + for (int i = 0; i < FdwXactCtl->num_xacts; i++) + { + FdwXactState fdwxact = FdwXactCtl->xacts[i]; + char *buf; + + buf = ProcessFdwXactBuffer(fdwxact->data.xid, fdwxact->data.umid, + fdwxact->insert_start_lsn, fdwxact->ondisk); + + if (buf == NULL) + continue; + + if (TransactionIdPrecedes(fdwxact->data.xid, result)) + result = fdwxact->data.xid; + + pfree(buf); + } + LWLockRelease(FdwXactLock); + + return result; +} + +/* + * Scan pg_fdwxact and fill FdwXactState depending on the on-disk data. + * This is called once at the beginning of recovery, saving any extra + * lookups in the future. FdwXactState files that are newer than the + * minimum XID horizon are discarded on the way. + */ +void +RestoreFdwXactData(void) +{ + DIR *cldir; + struct dirent *clde; + + LWLockAcquire(FdwXactLock, LW_EXCLUSIVE); + cldir = AllocateDir(FDWXACTS_DIR); + while ((clde = ReadDir(cldir, FDWXACTS_DIR)) != NULL) + { + if (strlen(clde->d_name) == FDWXACT_FILE_NAME_LEN && + strspn(clde->d_name, "0123456789ABCDEF_") == FDWXACT_FILE_NAME_LEN) + { + TransactionId xid; + Oid umid; + char *buf; + + sscanf(clde->d_name, "%08x_%08x", &xid, &umid); + + /* Read fdwxact data from disk */ + buf = ProcessFdwXactBuffer(xid, umid, InvalidXLogRecPtr, + true); + if (buf == NULL) + continue; + + /* Add this entry into the table of foreign transactions */ + FdwXactRedoAdd(buf, InvalidXLogRecPtr, InvalidXLogRecPtr); + } + } + + LWLockRelease(FdwXactLock); + FreeDir(cldir); +} + +/* + * Scan the shared memory entries of FdwXactState and valid them. + * + * This is run at the end of recovery, but before we allow backends to write + * WAL. + */ +void +RecoverFdwXacts(void) +{ + LWLockAcquire(FdwXactLock, LW_EXCLUSIVE); + for (int i = 0; i < FdwXactCtl->num_xacts; i++) + { + FdwXactState fdwxact = FdwXactCtl->xacts[i]; + char *buf; + + buf = ProcessFdwXactBuffer(fdwxact->data.xid, fdwxact->data.umid, + fdwxact->insert_start_lsn, fdwxact->ondisk); + + if (buf == NULL) + continue; + + ereport(LOG, + (errmsg("recovering foreign prepared transaction %s from shared memory", + fdwxact->data.identifier))); + + /* recovered, so reset the flag for entries generated by redo */ + fdwxact->inredo = false; + fdwxact->valid = true; + pfree(buf); + } + LWLockRelease(FdwXactLock); +} + +/* + * Store pointer to the start/end of the WAL record along with the xid in + * a fdwxact entry in shared memory FdwXactData structure. + */ +static void +FdwXactRedoAdd(char *buf, XLogRecPtr start_lsn, XLogRecPtr end_lsn) +{ + FdwXactStateOnDiskData *fdwxact_data = (FdwXactStateOnDiskData *) buf; + FdwXactState fdwxact; + + Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE)); + Assert(RecoveryInProgress()); + + /* + * Add this entry into the table of foreign transactions. The status of + * the transaction is set as preparing, since we do not know the exact + * status right now. Resolver will set it later based on the status of + * local transaction which prepared this foreign transaction. + */ + fdwxact = insert_fdwxact(fdwxact_data->dbid, fdwxact_data->xid, + fdwxact_data->umid, fdwxact_data->serverid, + fdwxact_data->owner, fdwxact_data->identifier); + + elog(DEBUG2, "added fdwxact entry in shared memory for foreign transaction, db %u xid %u user mapping %u owner %u id %s", + fdwxact_data->dbid, fdwxact_data->xid, + fdwxact_data->umid, fdwxact_data->owner, + fdwxact_data->identifier); + + /* + * Set status as PREPARED, since we do not know the xact status right now. + * We will set it later based on the status of local transaction that + * prepared this fdwxact entry. + */ + fdwxact->status = FDWXACT_STATUS_PREPARED; + fdwxact->insert_start_lsn = start_lsn; + fdwxact->insert_end_lsn = end_lsn; + fdwxact->inredo = true; /* added in redo */ + fdwxact->valid = false; + fdwxact->ondisk = XLogRecPtrIsInvalid(start_lsn); +} + +/* + * Remove the corresponding fdwxact entry from FdwXactCtl. Also remove + * FdwXactState file if a foreign transaction was saved via an earlier checkpoint. + * We could not found the FdwXactState entry in the case where a crash recovery + * starts from the point where is after added but before removed the entry. + */ +static void +FdwXactRedoRemove(TransactionId xid, Oid umid, bool givewarning) +{ + FdwXactState fdwxact; + int i; + + Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE)); + Assert(RecoveryInProgress()); + + for (i = 0; i < FdwXactCtl->num_xacts; i++) + { + fdwxact = FdwXactCtl->xacts[i]; + + if (fdwxact->data.xid == xid && fdwxact->data.umid == umid) + break; + } + + if (i >= FdwXactCtl->num_xacts) + return; + + /* Clean up entry and any files we may have left */ + remove_fdwxact(fdwxact); + + elog(DEBUG2, "removed fdwxact entry from shared memory for foreign transaction %s", + fdwxact->data.identifier); +} + +/* + * Reads foreign transaction data from xlog. During checkpoint this data will + * be moved to fdwxact files and ReadFdwXactStateFile should be used instead. + * + * Note clearly that this function accesses WAL during normal operation, similarly + * to the way WALSender or Logical Decoding would do. It does not run during + * crash recovery or standby processing. + */ +static void +XlogReadFdwXactData(XLogRecPtr lsn, char **buf, int *len) +{ + XLogRecord *record; + XLogReaderState *xlogreader; + char *errormsg; + TimeLineID save_currtli = ThisTimeLineID; + + xlogreader = XLogReaderAllocate(wal_segment_size, NULL, + XL_ROUTINE(.page_read = &read_local_xlog_page, + .segment_open = &wal_segment_open, + .segment_close = &wal_segment_close), + NULL); + + if (!xlogreader) + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"), + errdetail("Failed while allocating an XLog reading processor."))); + + XLogBeginRead(xlogreader, lsn); + record = XLogReadRecord(xlogreader, &errormsg); + + /* + * Restore immediately the timeline where it was previously, as + * read_local_xlog_page() could have changed it if the record was read + * while recovery was finishing or if the timeline has jumped in-between. + */ + ThisTimeLineID = save_currtli; + + if (record == NULL) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read foreign transaction state from xlog at %X/%X", + LSN_FORMAT_ARGS(lsn)))); + + if (XLogRecGetRmid(xlogreader) != RM_FDWXACT_ID || + (XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDWXACT_INSERT) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("expected foreign transaction state data is not present in xlog at %X/%X", + LSN_FORMAT_ARGS(lsn)))); + + if (len != NULL) + *len = XLogRecGetDataLen(xlogreader); + + *buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader)); + memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader)); + + XLogReaderFree(xlogreader); +} + +/* + * Given a transaction id, userid and serverid read it either from disk + * or read it directly via shmem xlog record pointer using the provided + * "insert_start_lsn". + */ +static char * +ProcessFdwXactBuffer(TransactionId xid, Oid umid, XLogRecPtr insert_start_lsn, + bool fromdisk) +{ + TransactionId origNextXid = + XidFromFullTransactionId(ShmemVariableCache->nextXid); + char *buf; + + Assert(LWLockHeldByMeInMode(FdwXactLock, LW_EXCLUSIVE)); + + if (!fromdisk) + Assert(!XLogRecPtrIsInvalid(insert_start_lsn)); + + /* Reject XID if too new */ + if (TransactionIdFollowsOrEquals(xid, origNextXid)) + { + if (fromdisk) + { + ereport(WARNING, + (errmsg("removing future fdwxact state file for xid %u and user mapping %u", + xid, umid))); + RemoveFdwXactStateFile(xid, umid, true); + } + else + { + ereport(WARNING, + (errmsg("removing future fdwxact state from memory for xid %u and user mapping %u", + xid, umid))); + FdwXactRedoRemove(xid, umid, true); + } + return NULL; + } + + if (fromdisk) + { + /* Read and validate file */ + buf = ReadFdwXactStateFile(xid, umid); + } + else + { + /* Read xlog data */ + XlogReadFdwXactData(insert_start_lsn, &buf, NULL); + } + + return buf; +} + +/* + * Read and validate the foreign transaction state file. + * + * If it looks OK (has a valid magic number and CRC), return the palloc'd + * contents of the file, issuing an error when finding corrupted data. + * This state can be reached when doing recovery. + */ +static char * +ReadFdwXactStateFile(TransactionId xid, Oid umid) +{ + char path[MAXPGPATH]; + int fd; + FdwXactStateOnDiskData *fdwxact_file_data; + struct stat stat; + uint32 crc_offset; + pg_crc32c calc_crc; + pg_crc32c file_crc; + char *buf; + int r; + + FdwXactStateFilePath(path, xid, umid); + + fd = OpenTransientFile(path, O_RDONLY | PG_BINARY); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open FDW transaction state file \"%s\": %m", + path))); + + /* + * Check file length. We can determine a lower bound pretty easily. We + * set an upper bound to avoid palloc() failure on a corrupt file, though + * we can't guarantee that we won't get an out of memory error anyway, + * even on a valid file. + */ + if (fstat(fd, &stat)) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not stat FDW transaction state file \"%s\": %m", + path))); + + if (stat.st_size < (offsetof(FdwXactStateOnDiskData, identifier) + + sizeof(pg_crc32c)) || + stat.st_size > MaxAllocSize) + + ereport(ERROR, + (errcode_for_file_access(), + errmsg("too large FDW transaction state file \"%s\": %m", + path))); + + crc_offset = stat.st_size - sizeof(pg_crc32c); + if (crc_offset != MAXALIGN(crc_offset)) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("incorrect alignment of CRC offset for file \"%s\"", + path))); + + /* + * Ok, slurp in the file. + */ + buf = (char *) palloc(stat.st_size); + fdwxact_file_data = (FdwXactStateOnDiskData *) buf; + + /* Slurp the file */ + pgstat_report_wait_start(WAIT_EVENT_FDWXACT_FILE_READ); + r = read(fd, buf, stat.st_size); + if (r != stat.st_size) + { + if (r < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read file \"%s\": %m", path))); + else + ereport(ERROR, + (errmsg("could not read file \"%s\": read %d of %zu", + path, r, (Size) stat.st_size))); + } + pgstat_report_wait_end(); + + if (CloseTransientFile(fd)) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not close file \"%s\": %m", path))); + + /* + * Check the CRC. + */ + INIT_CRC32C(calc_crc); + COMP_CRC32C(calc_crc, buf, crc_offset); + FIN_CRC32C(calc_crc); + + file_crc = *((pg_crc32c *) (buf + crc_offset)); + + if (!EQ_CRC32C(calc_crc, file_crc)) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("calculated CRC checksum does not match value stored in file \"%s\"", + path))); + + /* Check if the contents is an expected data */ + fdwxact_file_data = (FdwXactStateOnDiskData *) buf; + if (fdwxact_file_data->xid != xid || + fdwxact_file_data->umid != umid) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("invalid foreign transaction state file \"%s\"", + path))); + + return buf; +} + +/* + * Remove the foreign transaction file for given entry. + * + * If giveWarning is false, do not complain about file-not-present; + * this is an expected case during WAL replay. + */ +static void +RemoveFdwXactStateFile(TransactionId xid, Oid umid, bool giveWarning) +{ + char path[MAXPGPATH]; + + FdwXactStateFilePath(path, xid, umid); + if (unlink(path) < 0 && (errno != ENOENT || giveWarning)) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove foreign transaction state file \"%s\": %m", + path))); +} + +/* + * Return the list of FdwXactState entries that match at least one given criteria + * that is not invalid. The caller must hold FdwXactLock. + */ +static List * +find_fdwxacts(TransactionId xid, Oid umid, Oid dbid) +{ + List *res = NIL; + + Assert(LWLockHeldByMe(FdwXactLock)); + + for (int i = 0; i < FdwXactCtl->num_xacts; i++) + { + FdwXactState fdwxact = FdwXactCtl->xacts[i]; + bool match = false; + + if (!fdwxact->valid) + continue; + + /* xid */ + if (TransactionIdIsValid(xid) && xid == fdwxact->data.xid) + match = true; + + /* umid */ + if (OidIsValid(umid) && umid == fdwxact->data.umid) + match = true; + + /* dbid */ + if (OidIsValid(dbid) && dbid == fdwxact->data.dbid) + match = true; + + if (match) + res = lappend(res, fdwxact); + } + + return res; +} + +/* + * Get FdwXact entry and do some sanity checks. If check_twophase_xact is true, we + * also check if the given xid is prepared. The caller must hold FdwXactLock. + */ +static FdwXactState +get_fdwxact_with_check(TransactionId xid, Oid umid, bool check_twophase_xact) +{ + FdwXactState fdwxact = NULL; + Oid myuserid; + + Assert(LWLockHeldByMe(FdwXactLock)); + + /* Look for FdwXactState entry that matches the given xid and umid */ + for (int i = 0; i < FdwXactCtl->num_xacts; i++) + { + FdwXactState fx = FdwXactCtl->xacts[i]; + + if (fx->valid && fx->data.xid == xid && fx->data.umid == umid) + { + fdwxact = fx; + break; + } + } + + /* not found */ + if (!fdwxact) + return NULL; + + /* check if belonging to another database */ + if (fdwxact->data.dbid != MyDatabaseId) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("foreign transaction belongs to another database"), + errhint("Connect to the database where the transaction was created to finish it."))); + + /* permission check */ + myuserid = GetUserId(); + if (myuserid != fdwxact->data.owner && !superuser_arg(myuserid)) + ereport(ERROR, + (errmsg("permission denied to resolve prepared foreign transaction"), + errhint("Must be superuser or the user that prepared the transaction"))); + + /* check if the entry is being processed by someone */ + if (fdwxact->locking_backend != InvalidBackendId) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("foreign transaction with transaction identifier \"%s\" is busy", + fdwxact->data.identifier))); + + if (check_twophase_xact && TwoPhaseExists(fdwxact->data.xid)) + { + /* + * the entry's local transaction is prepared. Since we cannot know the + * fate of the local transaction, we cannot resolve this foreign + * transaction. + */ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot resolve foreign transaction with identifier \"%s\" whose local transaction is in-progress", + fdwxact->data.identifier), + errhint("Do COMMIT PREPARED or ROLLBACK PREPARED"))); + } + + return fdwxact; +} + +/* Error cleanup callback for pg_foreign_resolve/remove_xact */ +static void +pg_foreign_xact_callback(int code, Datum arg) +{ + FdwXactState fdwxact = (FdwXactState) DatumGetPointer(arg); + + if (fdwxact->valid) + { + Assert(fdwxact->locking_backend == MyBackendId); + + LWLockAcquire(FdwXactLock, LW_EXCLUSIVE); + fdwxact->locking_backend = InvalidBackendId; + LWLockRelease(FdwXactLock); + } +} + +/* Built in functions */ + +/* + * Structure to hold and iterate over the foreign transactions to be displayed + * by the built-in functions. + */ +typedef struct +{ + FdwXactState fdwxacts; + int num_xacts; + int cur_xact; +} WorkingStatus; + +Datum +pg_foreign_xacts(PG_FUNCTION_ARGS) +{ +#define PG_PREPARED_FDWXACTS_COLS 7 + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + TupleDesc tupdesc; + Tuplestorestate *tupstore; + MemoryContext per_query_ctx; + MemoryContext oldcontext; + + /* check to see if caller supports us returning a tuplestore */ + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that cannot accept a set"))); + if (!(rsinfo->allowedModes & SFRM_Materialize)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("materialize mode required, but it is not allowed in this context"))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; + oldcontext = MemoryContextSwitchTo(per_query_ctx); + + tupstore = tuplestore_begin_heap(true, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + + MemoryContextSwitchTo(oldcontext); + + LWLockAcquire(FdwXactLock, LW_SHARED); + for (int i = 0; i < FdwXactCtl->num_xacts; i++) + { + FdwXactState fdwxact = FdwXactCtl->xacts[i]; + FdwXactStatus status; + char *xact_status; + Datum values[PG_PREPARED_FDWXACTS_COLS]; + bool nulls[PG_PREPARED_FDWXACTS_COLS]; + + if (!fdwxact->valid) + continue; + + memset(nulls, 0, sizeof(nulls)); + + SpinLockAcquire(&fdwxact->mutex); + status = fdwxact->status; + SpinLockRelease(&fdwxact->mutex); + + values[0] = TransactionIdGetDatum(fdwxact->data.xid); + values[1] = ObjectIdGetDatum(fdwxact->data.umid); + values[2] = ObjectIdGetDatum(fdwxact->data.owner); + values[3] = ObjectIdGetDatum(fdwxact->data.dbid); + + switch (status) + { + case FDWXACT_STATUS_PREPARING: + xact_status = "preparing"; + break; + case FDWXACT_STATUS_PREPARED: + xact_status = "prepared"; + break; + case FDWXACT_STATUS_COMMITTING: + xact_status = "committing"; + break; + case FDWXACT_STATUS_ABORTING: + xact_status = "aborting"; + break; + default: + xact_status = "unknown"; + break; + } + + values[4] = CStringGetTextDatum(xact_status); + values[5] = CStringGetTextDatum(fdwxact->data.identifier); + + if (fdwxact->locking_backend != InvalidBackendId) + { + PGPROC *locker = BackendIdGetProc(fdwxact->locking_backend); + + values[6] = Int32GetDatum(locker->pid); + } + else + nulls[6] = true; + + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + LWLockRelease(FdwXactLock); + + /* clean up and return the tuplestore */ + tuplestore_donestoring(tupstore); + + return (Datum) 0; +} + +/* + * Built-in SQL function to resolve a prepared foreign transaction. + */ +Datum +pg_resolve_foreign_xact(PG_FUNCTION_ARGS) +{ + TransactionId xid = DatumGetTransactionId(PG_GETARG_DATUM(0)); + Oid umid = PG_GETARG_OID(1); + FdwXactState fdwxact; + + LWLockAcquire(FdwXactLock, LW_EXCLUSIVE); + + fdwxact = get_fdwxact_with_check(xid, umid, true); + + /* lock it */ + fdwxact->locking_backend = MyBackendId; + + LWLockRelease(FdwXactLock); + + /* + * Resolve the foreign transaction. We ensure unlocking FdwXact entry at + * an error or an interruption. + * + * XXX we assume that an interruption doesn't happen between locking + * FdwXact entry and registering the callback, especially in + * LWLockRelease(). + */ + PG_ENSURE_ERROR_CLEANUP(pg_foreign_xact_callback, + (Datum) PointerGetDatum(fdwxact)); + { + ResolveOneFdwXact(fdwxact); + } + PG_END_ENSURE_ERROR_CLEANUP(pg_foreign_xact_callback, + (Datum) PointerGetDatum(fdwxact)); + + PG_RETURN_BOOL(true); +} + +/* + * Built-in function to remove a prepared foreign transaction entry without + * resolution. The function gives a way to forget about such prepared + * transaction in case: the foreign server where it is prepared is no longer + * available, the user which prepared this transaction needs to be dropped. + */ +Datum +pg_remove_foreign_xact(PG_FUNCTION_ARGS) +{ + TransactionId xid = DatumGetTransactionId(PG_GETARG_DATUM(0)); + Oid umid = PG_GETARG_OID(1); + FdwXactState fdwxact; + + LWLockAcquire(FdwXactLock, LW_EXCLUSIVE); + + fdwxact = get_fdwxact_with_check(xid, umid, false); + + /* Clean up entry and any files we may have left */ + remove_fdwxact(fdwxact); + + LWLockRelease(FdwXactLock); + + PG_RETURN_BOOL(true); } diff --git a/src/backend/access/transam/fdwxact_launcher.c b/src/backend/access/transam/fdwxact_launcher.c new file mode 100644 index 0000000000..59f061773c --- /dev/null +++ b/src/backend/access/transam/fdwxact_launcher.c @@ -0,0 +1,587 @@ +/*------------------------------------------------------------------------- + * + * launcher.c + * + * The foreign transaction resolver launcher process starts foreign + * transaction resolver processes. The launcher schedules resolver + * process to be started when requested by backend process. + * + * Portions Copyright (c) 2021, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/access/transam/fdwxact_launcher.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "funcapi.h" +#include "pgstat.h" +#include "funcapi.h" + +#include "access/fdwxact.h" +#include "access/fdwxact_launcher.h" +#include "access/fdwxact_resolver.h" +#include "access/resolver_internal.h" +#include "access/twophase.h" +#include "commands/dbcommands.h" +#include "funcapi.h" +#include "postmaster/bgworker.h" +#include "postmaster/interrupt.h" +#include "storage/ipc.h" +#include "storage/proc.h" +#include "tcop/tcopprot.h" +#include "utils/builtins.h" + +/* max sleep time between cycles (3min) */ +#define DEFAULT_NAPTIME_PER_CYCLE 180000L + +FdwXactResolver *MyFdwXactResolver = NULL; + +static volatile sig_atomic_t got_SIGUSR2 = false; + +static void FdwXactLauncherOnExit(int code, Datum arg); +static void LaunchFdwXactResolver(Oid dbid, Oid serverid); +static bool RelaunchFdwXactResolvers(void); +static FdwXactResolver *FdwXactResolverFind(Oid dbid, Oid serverid); + +/* Signal handler */ +static void FdwXactLaunchHandler(SIGNAL_ARGS); + + +/* + * Wake up the launcher process to request launching new resolvers + * immediately. + */ +void +RequestToLaunchFdwXactResolver(void) +{ + if (FdwXactResolverCtl->launcher_pid != InvalidPid) + kill(FdwXactResolverCtl->launcher_pid, SIGUSR2); +} + +/* Report shared memory space needed by FdwXactLauncherShmemInit */ +Size +FdwXactLauncherShmemSize(void) +{ + Size size = 0; + + size = add_size(size, SizeOfFdwXactResolverCtlData); + size = add_size(size, mul_size(max_foreign_xact_resolvers, + sizeof(FdwXactResolver))); + + return size; +} + +/* + * Allocate and initialize foreign transaction resolver shared + * memory. + */ +void +FdwXactLauncherShmemInit(void) +{ + bool found; + + FdwXactResolverCtl = ShmemInitStruct("Foreign Transaction Launcher Data", + FdwXactLauncherShmemSize(), + &found); + + if (!IsUnderPostmaster) + { + int slot; + + /* First time through, so initialize */ + MemSet(FdwXactResolverCtl, 0, FdwXactLauncherShmemSize()); + FdwXactResolverCtl->launcher_pid = InvalidPid; + + for (slot = 0; slot < max_foreign_xact_resolvers; slot++) + { + FdwXactResolver *resolver = &FdwXactResolverCtl->resolvers[slot]; + + memset(resolver, 0, sizeof(FdwXactResolver)); + SpinLockInit(&(resolver->mutex)); + } + } +} + +/* + * Cleanup function for fdwxact launcher + * + * Called on fdwxact launcher exit. + */ +static void +FdwXactLauncherOnExit(int code, Datum arg) +{ + FdwXactResolverCtl->launcher_pid = InvalidPid; +} + +/* SIGUSR2: set flag to launch new resolver process immediately */ +static void +FdwXactLaunchHandler(SIGNAL_ARGS) +{ + int save_errno = errno; + + got_SIGUSR2 = true; + SetLatch(MyLatch); + + errno = save_errno; +} + +/* + * Main loop for the fdwxact launcher process. + */ +void +FdwXactLauncherMain(Datum main_arg) +{ + TimestampTz last_start_time = 0; + + ereport(DEBUG1, + (errmsg("fdwxact resolver launcher started"))); + + before_shmem_exit(FdwXactLauncherOnExit, (Datum) 0); + + Assert(FdwXactResolverCtl->launcher_pid == InvalidPid); + FdwXactResolverCtl->launcher_pid = MyProcPid; + FdwXactResolverCtl->launcher_latch = &MyProc->procLatch; + + pqsignal(SIGHUP, SignalHandlerForConfigReload); + pqsignal(SIGUSR2, FdwXactLaunchHandler); + pqsignal(SIGTERM, die); + BackgroundWorkerUnblockSignals(); + + BackgroundWorkerInitializeConnection(NULL, NULL, 0); + + /* Enter main loop */ + for (;;) + { + TimestampTz now; + long wait_time = DEFAULT_NAPTIME_PER_CYCLE; + int rc; + + CHECK_FOR_INTERRUPTS(); + ResetLatch(MyLatch); + + now = GetCurrentTimestamp(); + + /* + * Limit the start retry to once a + * foreign_xact_resolution_retry_interval but always attempt to start + * when requested. + */ + if (got_SIGUSR2 || + TimestampDifferenceExceeds(last_start_time, now, + foreign_xact_resolution_retry_interval)) + { + MemoryContext oldctx; + MemoryContext subctx; + bool launched; + + if (got_SIGUSR2) + got_SIGUSR2 = false; + + subctx = AllocSetContextCreate(TopMemoryContext, + "Foreign Transaction Launcher", + ALLOCSET_DEFAULT_SIZES); + oldctx = MemoryContextSwitchTo(subctx); + + launched = RelaunchFdwXactResolvers(); + if (launched) + { + last_start_time = now; + wait_time = foreign_xact_resolution_retry_interval; + } + + /* Switch back to original memory context. */ + MemoryContextSwitchTo(oldctx); + + /* Clean the temporary memory. */ + MemoryContextDelete(subctx); + } + else + { + /* + * The wait in previous cycle was interrupted in less than + * foreign_xact_resolution_retry_interval since last resolver + * started, this usually means crash of the resolver, so we should + * retry in foreign_xact_resolution_retry_interval again. + */ + wait_time = foreign_xact_resolution_retry_interval; + } + + /* Wait for more work */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + wait_time, + WAIT_EVENT_FDWXACT_LAUNCHER_MAIN); + + if (rc & WL_POSTMASTER_DEATH) + proc_exit(1); + + if (rc & WL_LATCH_SET) + { + ResetLatch(MyLatch); + CHECK_FOR_INTERRUPTS(); + } + + if (ConfigReloadPending) + { + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + } + } + + /* Not reachable */ +} + +/* + * Request the launcher to launch or wakeup foreign transaction resolvers for + * the given servers. + */ +void +LaunchOrWakeupFdwXactResolver(List *serveroids_orig) +{ + List *wakeup_resolvers = NIL; + List *serveroids = list_copy(serveroids_orig); + + /* Collect running resolvers to wakeup */ + LWLockAcquire(FdwXactResolverLock, LW_SHARED); + for (int i = 0; + i < max_foreign_xact_resolvers && list_length(serveroids) > 0; + i++) + { + FdwXactResolver *resolver = &FdwXactResolverCtl->resolvers[i]; + + if (resolver->in_use && + resolver->dbid == MyDatabaseId && + list_member_oid(serveroids, resolver->serverid)) + { + wakeup_resolvers = lappend(wakeup_resolvers, resolver); + serveroids = list_delete_oid(serveroids, resolver->serverid); + } + } + LWLockRelease(FdwXactResolverLock); + + if (wakeup_resolvers != NIL) + { + ListCell *lc; + + foreach(lc, wakeup_resolvers) + { + FdwXactResolver *resolver = (FdwXactResolver *) lfirst(lc); + + /* + * Wakeup the resolver. It's possible that the resolver is + * starting up and doesn't attach its slot yet. Since the resolver + * will find FdwXactState entry we inserted soon we don't + * anything. + */ + if (resolver->latch) + SetLatch(resolver->latch); + } + + list_free(wakeup_resolvers); + } + + /* Request to launch new resolvers if any */ + if (list_length(serveroids) > 0) + RequestToLaunchFdwXactResolver(); +} + +/* + * Launch a foreign transaction resolver process that will connect to given + * 'dbid' and 'serverid'. + */ +static void +LaunchFdwXactResolver(Oid dbid, Oid serverid) +{ + BackgroundWorker bgw; + BackgroundWorkerHandle *bgw_handle; + FdwXactResolver *resolver; + int unused_slot = -1; + int i; + + LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE); + + /* Find unused resolver slot */ + for (i = 0; i < max_foreign_xact_resolvers; i++) + { + FdwXactResolver *resolver = &FdwXactResolverCtl->resolvers[i]; + + if (!resolver->in_use) + { + unused_slot = i; + break; + } + } + + /* No unused found */ + if (i >= max_foreign_xact_resolvers) + ereport(ERROR, + (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), + errmsg("out of foreign transaction resolver slots"), + errhint("You might need to increase max_foreign_transaction_resolvers."))); + + resolver = &FdwXactResolverCtl->resolvers[unused_slot]; + resolver->in_use = true; + resolver->dbid = dbid; + resolver->serverid = serverid; + LWLockRelease(FdwXactResolverLock); + + /* Register the new dynamic worker */ + memset(&bgw, 0, sizeof(bgw)); + bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | + BGWORKER_BACKEND_DATABASE_CONNECTION; + bgw.bgw_start_time = BgWorkerStart_RecoveryFinished; + snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres"); + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactResolverMain"); + snprintf(bgw.bgw_name, BGW_MAXLEN, + "foreign transaction resolver for server %u on database %u", + resolver->serverid, resolver->dbid); + snprintf(bgw.bgw_type, BGW_MAXLEN, "foreign transaction resolver"); + bgw.bgw_restart_time = BGW_NEVER_RESTART; + bgw.bgw_notify_pid = MyProcPid; + bgw.bgw_main_arg = Int32GetDatum(unused_slot); + + if (!RegisterDynamicBackgroundWorker(&bgw, &bgw_handle)) + { + /* Failed to launch, cleanup the worker slot */ + SpinLockAcquire(&(MyFdwXactResolver->mutex)); + resolver->in_use = false; + SpinLockRelease(&(MyFdwXactResolver->mutex)); + + ereport(WARNING, + (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), + errmsg("out of background worker slots"), + errhint("You might need to increase max_worker_processes."))); + } + + /* + * We don't need to wait until it attaches here because we're going to + * wait until all foreign transactions are resolved. + */ +} + +/* + * Check the pending foreign transaction entries and (re)launch the resolvers + * for them if necessary. Return true if launched at least one resolver. + */ +static bool +RelaunchFdwXactResolvers(void) +{ + HTAB *launch_resolvers; + HASHCTL ctl; + HASH_SEQ_STATUS status; + struct fdwxact_resolver_entry + { + Oid dbid; + Oid serverid; + }; + struct fdwxact_resolver_entry *entry; + + /* Create a hash map for resolvers that need to launch */ + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(struct fdwxact_resolver_entry); + ctl.entrysize = sizeof(struct fdwxact_resolver_entry); + launch_resolvers = hash_create("fdwxact resolvers to launch", + 32, &ctl, HASH_ELEM | HASH_BLOBS); + + LWLockAcquire(FdwXactLock, LW_SHARED); + for (int i = 0; i < FdwXactCtl->num_xacts; i++) + { + FdwXactState fdwxact = FdwXactCtl->xacts[i]; + + if (!fdwxact->valid) + continue; + + /* + * We need to launch resolver process if the foreign transaction is + * not held by anyone and is not a part of the local prepared + * transaction. + */ + if (fdwxact->locking_backend == InvalidBackendId && + !TwoPhaseExists(fdwxact->data.xid)) + { + struct fdwxact_resolver_entry ent = { + .dbid = fdwxact->data.dbid, + .serverid = fdwxact->data.serverid + }; + + hash_search(launch_resolvers, (void *) &ent, HASH_ENTER, NULL); + } + } + LWLockRelease(FdwXactLock); + + /* There is no foreign transaction to resolve, no need to launch new one */ + if (hash_get_num_entries(launch_resolvers) == 0) + { + hash_destroy(launch_resolvers); + return false; + } + + /* Check if the resolver is already running */ + LWLockAcquire(FdwXactResolverLock, LW_SHARED); + for (int i = 0; i < max_foreign_xact_resolvers; i++) + { + FdwXactResolver *resolver = &FdwXactResolverCtl->resolvers[i]; + struct fdwxact_resolver_entry ent; + + if (!resolver->in_use) + continue; + + ent.dbid = resolver->dbid; + ent.serverid = resolver->serverid; + hash_search(launch_resolvers, &ent, HASH_REMOVE, NULL); + } + LWLockRelease(FdwXactResolverLock); + + /* Return if resolvers for all foreign transactions are already running */ + if (hash_get_num_entries(launch_resolvers) == 0) + { + hash_destroy(launch_resolvers); + return false; + } + + /* Launch new resolvers */ + hash_seq_init(&status, launch_resolvers); + while ((entry = (struct fdwxact_resolver_entry *) hash_seq_search(&status)) != NULL) + LaunchFdwXactResolver(entry->dbid, entry->serverid); + + hash_destroy(launch_resolvers); + return true; +} + +/* Register a background worker running the foreign transaction launcher */ +void +FdwXactLauncherRegister(void) +{ + BackgroundWorker bgw; + + if (max_foreign_xact_resolvers == 0) + return; + + memset(&bgw, 0, sizeof(bgw)); + bgw.bgw_flags = BGWORKER_SHMEM_ACCESS | + BGWORKER_BACKEND_DATABASE_CONNECTION; + bgw.bgw_start_time = BgWorkerStart_RecoveryFinished; + snprintf(bgw.bgw_library_name, BGW_MAXLEN, "postgres"); + snprintf(bgw.bgw_function_name, BGW_MAXLEN, "FdwXactLauncherMain"); + snprintf(bgw.bgw_name, BGW_MAXLEN, + "foreign transaction launcher"); + snprintf(bgw.bgw_type, BGW_MAXLEN, + "foreign transaction launcher"); + bgw.bgw_restart_time = 5; + bgw.bgw_notify_pid = 0; + bgw.bgw_main_arg = (Datum) 0; + + RegisterBackgroundWorker(&bgw); +} + +bool +IsFdwXactLauncher(void) +{ + return FdwXactResolverCtl->launcher_pid == MyProcPid; +} + +/* + * Return the foreign transaction resolver running for dbid and serverid + */ +FdwXactResolver * +FdwXactResolverFind(Oid dbid, Oid serverid) +{ + Assert(LWLockHeldByMe(FdwXactResolverLock)); + for (int i = 0; i < max_foreign_xact_resolvers; i++) + { + FdwXactResolver *resolver = &FdwXactResolverCtl->resolvers[i]; + + if (resolver->in_use && resolver->dbid == dbid && + resolver->serverid == serverid) + return resolver; + } + + return NULL; +} + +/* + * Stop the given foreign transaction resolver, and wait until it detaches + * from slot. + */ +void +StopFdwXactResolver(Oid dbid, Oid serverid) +{ + FdwXactResolver *resolver; + + LWLockAcquire(FdwXactResolverLock, LW_SHARED); + + resolver = FdwXactResolverFind(dbid, serverid); + + /* Not worker, nothing to do */ + if (!resolver) + { + LWLockRelease(FdwXactResolverLock); + return; + } + + /* Found the resolver, terminate it ... */ + kill(resolver->pid, SIGTERM); + + /* ... and wait for it to die */ + for (;;) + { + int rc; + + /* is it gone? */ + if (!resolver->in_use) + break; + + LWLockRelease(FdwXactResolverLock); + + /* Wait a bit --- we don't expect to have to wait long. */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + 10L, WAIT_EVENT_BGWORKER_SHUTDOWN); + + if (rc & WL_LATCH_SET) + { + ResetLatch(MyLatch); + CHECK_FOR_INTERRUPTS(); + } + + LWLockAcquire(FdwXactResolverLock, LW_SHARED); + } + + LWLockRelease(FdwXactResolverLock); +} + +/* + * Stop the fdwxact resolver running for the given foreign server. + */ +Datum +pg_stop_foreign_xact_resolver(PG_FUNCTION_ARGS) +{ + char *servername = text_to_cstring(PG_GETARG_TEXT_P(0)); + ForeignServer *server; + FdwXactResolver *resolver; + + /* Must be super user */ + if (!superuser()) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied to stop foreign transaction resolver"))); + + server = GetForeignServerByName(servername, false); + + LWLockAcquire(FdwXactResolverLock, LW_SHARED); + resolver = FdwXactResolverFind(MyDatabaseId, server->serverid); + LWLockRelease(FdwXactResolverLock); + + if (!resolver) + ereport(ERROR, + (errmsg("there is no running foreign transaction resolver process on server %s", + servername))); + + StopFdwXactResolver(MyDatabaseId, server->serverid); + pfree(server); + + PG_RETURN_BOOL(true); +} diff --git a/src/backend/access/transam/fdwxact_resolver.c b/src/backend/access/transam/fdwxact_resolver.c new file mode 100644 index 0000000000..587c4759ca --- /dev/null +++ b/src/backend/access/transam/fdwxact_resolver.c @@ -0,0 +1,339 @@ +/*------------------------------------------------------------------------- + * + * resolver.c + * + * The foreign transaction resolver background worker resolves in-doubt + * foreign transactions, foreign transactions participate to a distributed + * transaction but aren't being processed anyone. A resolver process is + * launched per foreign server by foreign transaction launcher. + * + * Normal termination is by SIGTERM, which instructs the resolver process + * to exit(0) at the next convenient moment. Emergency termination is by + * SIGQUIT; like any backend. The resolver process also terminate by timeouts + * only if there is no pending foreign transactions on the database waiting + * to be resolved. + * + * Portions Copyright (c) 2021, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/backend/access/transam/fdwxact_resolver.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include +#include + +#include "access/fdwxact.h" +#include "access/fdwxact_resolver.h" +#include "access/fdwxact_launcher.h" +#include "access/resolver_internal.h" +#include "access/transam.h" +#include "access/twophase.h" +#include "access/xact.h" +#include "commands/dbcommands.h" +#include "funcapi.h" +#include "libpq/libpq.h" +#include "miscadmin.h" +#include "pgstat.h" +#include "postmaster/bgworker.h" +#include "postmaster/interrupt.h" +#include "storage/ipc.h" +#include "tcop/tcopprot.h" +#include "utils/builtins.h" +#include "utils/timeout.h" +#include "utils/timestamp.h" + +/* max sleep time between cycles (3min) */ +#define DEFAULT_NAPTIME_PER_CYCLE 180000L + +/* GUC parameters */ +int foreign_xact_resolution_retry_interval; +int foreign_xact_resolver_timeout = 60 * 1000; + +FdwXactResolverCtlData *FdwXactResolverCtl; + +static void FdwXactResolverLoop(void); +static long FdwXactResolverComputeSleepTime(TimestampTz now); +static void FdwXactResolverCheckTimeout(TimestampTz now); + +static void FdwXactResolverOnExit(int code, Datum arg); +static void FdwXactResolverDetach(void); +static void FdwXactResolverAttach(int slot); +static void FdwXactResolverProcessInDoubtXacts(void); + +/* The list of currently holding FdwXact entries. */ +static List *heldFdwXactEntries = NIL; + +static TimestampTz last_resolution_time = -1; + +/* + * Detach the resolver and cleanup the resolver info. + */ +static void +FdwXactResolverDetach(void) +{ + LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE); + + MyFdwXactResolver->pid = InvalidPid; + MyFdwXactResolver->in_use = false; + MyFdwXactResolver->dbid = InvalidOid; + MyFdwXactResolver->serverid = InvalidOid; + + LWLockRelease(FdwXactResolverLock); + + /* + * Force to send remaining WAL statistics to the stats collector at + * process exit. + * + * Since pgstat_send_wal is invoked with 'force' is false in main loop + * to avoid overloading to the stats collector, there may exist unsent + * stats counters for the WAL writer. + */ + pgstat_send_wal(true); +} + +/* + * Cleanup up foreign transaction resolver info and releas the holding + * FdwXactState entries. + */ +static void +FdwXactResolverOnExit(int code, Datum arg) +{ + ListCell *lc; + + FdwXactResolverDetach(); + + /* Release the held foreign transaction entries */ + LWLockAcquire(FdwXactLock, LW_EXCLUSIVE); + foreach(lc, heldFdwXactEntries) + { + FdwXactState fdwxact = (FdwXactState) lfirst(lc); + + if (fdwxact->valid && fdwxact->locking_backend == MyBackendId) + fdwxact->locking_backend = InvalidBackendId; + } + LWLockRelease(FdwXactLock); +} + +/* + * Attach to a slot. + */ +static void +FdwXactResolverAttach(int slot) +{ + /* Block concurrent access */ + LWLockAcquire(FdwXactResolverLock, LW_EXCLUSIVE); + + Assert(slot >= 0 && slot < max_foreign_xact_resolvers); + MyFdwXactResolver = &FdwXactResolverCtl->resolvers[slot]; + + if (!MyFdwXactResolver->in_use) + { + LWLockRelease(FdwXactResolverLock); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("foreign transaction resolver slot %d is empty, cannot attach", + slot))); + } + + Assert(OidIsValid(MyFdwXactResolver->dbid)); + Assert(OidIsValid(MyFdwXactResolver->serverid)); + + MyFdwXactResolver->pid = MyProcPid; + MyFdwXactResolver->latch = &MyProc->procLatch; + + before_shmem_exit(FdwXactResolverOnExit, (Datum) 0); + + LWLockRelease(FdwXactResolverLock); +} + +/* Foreign transaction resolver entry point */ +void +FdwXactResolverMain(Datum main_arg) +{ + int slot = DatumGetInt32(main_arg); + char *datname; + ForeignServer *server; + + /* Attach to a slot */ + FdwXactResolverAttach(slot); + + /* Establish signal handlers */ + pqsignal(SIGHUP, SignalHandlerForConfigReload); + pqsignal(SIGTERM, die); + BackgroundWorkerUnblockSignals(); + + /* Connect to our database */ + BackgroundWorkerInitializeConnectionByOid(MyFdwXactResolver->dbid, InvalidOid, 0); + + StartTransactionCommand(); + datname = get_database_name(MyFdwXactResolver->dbid); + server = GetForeignServer(MyFdwXactResolver->serverid); + ereport(LOG, + (errmsg("foreign transaction resolver for server \"%s\" on database \"%s\" has started", + server->servername, datname))); + pfree(datname); + pfree(server); + CommitTransactionCommand(); + + /* Initialize stats to a sanish value */ + last_resolution_time = GetCurrentTimestamp(); + + /* Run the main loop */ + FdwXactResolverLoop(); + + proc_exit(0); +} + +/* + * Fdwxact resolver main loop + */ +static void +FdwXactResolverLoop(void) +{ + /* Enter main loop */ + for (;;) + { + TimestampTz now; + int rc; + long sleep_time = DEFAULT_NAPTIME_PER_CYCLE; + + ResetLatch(MyLatch); + + CHECK_FOR_INTERRUPTS(); + + if (ConfigReloadPending) + { + ConfigReloadPending = false; + ProcessConfigFile(PGC_SIGHUP); + } + + /* Resolve in-doubt transactions if any */ + FdwXactResolverProcessInDoubtXacts(); + + now = GetCurrentTimestamp(); + FdwXactResolverCheckTimeout(now); + sleep_time = FdwXactResolverComputeSleepTime(now); + + /* Send WAL statistics to the stats collector */ + pgstat_send_wal(false); + + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + sleep_time, + WAIT_EVENT_FDWXACT_RESOLVER_MAIN); + + if (rc & WL_POSTMASTER_DEATH) + proc_exit(1); + } +} + +/* + * Check whether there have been foreign transactions by the backend within + * foreign_xact_resolver_timeout and shutdown if not. + */ +static void +FdwXactResolverCheckTimeout(TimestampTz now) +{ + TimestampTz timeout; + ForeignServer *server; + + if (foreign_xact_resolver_timeout == 0) + return; + + timeout = TimestampTzPlusMilliseconds(last_resolution_time, + foreign_xact_resolver_timeout); + + if (now < timeout) + return; + + /* Reached timeout, exit */ + StartTransactionCommand(); + server = GetForeignServer(MyFdwXactResolver->serverid); + ereport(LOG, + (errmsg("foreign transaction resolver for server \"%s\" on database \"%s\" will stop because the timeout", + server->servername, + get_database_name(MyDatabaseId)))); + CommitTransactionCommand(); + FdwXactResolverDetach(); + proc_exit(0); +} + +/* + * Compute how long we should sleep by the next cycle. We can sleep until the time + * out. + */ +static long +FdwXactResolverComputeSleepTime(TimestampTz now) +{ + long sleeptime = DEFAULT_NAPTIME_PER_CYCLE; + + if (foreign_xact_resolver_timeout > 0) + { + TimestampTz timeout; + + /* Compute relative time until wakeup. */ + timeout = TimestampTzPlusMilliseconds(last_resolution_time, + foreign_xact_resolver_timeout); + sleeptime = TimestampDifferenceMilliseconds(now, timeout); + } + + return sleeptime; +} + +bool +IsFdwXactResolver(void) +{ + return MyFdwXactResolver != NULL; +} + +/* + * Process in-doubt foreign transactions. + */ +static void +FdwXactResolverProcessInDoubtXacts(void) +{ + ListCell *lc; + + Assert(heldFdwXactEntries == NIL); + + /* Hold all in-doubt foreign transactions */ + LWLockAcquire(FdwXactLock, LW_EXCLUSIVE); + for (int i = 0; i < FdwXactCtl->num_xacts; i++) + { + FdwXactState fdwxact = FdwXactCtl->xacts[i]; + + if (fdwxact->valid && + fdwxact->locking_backend == InvalidBackendId && + fdwxact->data.dbid == MyFdwXactResolver->dbid && + fdwxact->data.serverid == MyFdwXactResolver->serverid && + !TwoPhaseExists(fdwxact->data.xid)) + { + fdwxact->locking_backend = MyBackendId; + heldFdwXactEntries = lappend(heldFdwXactEntries, fdwxact); + } + } + LWLockRelease(FdwXactLock); + + foreach(lc, heldFdwXactEntries) + { + FdwXactState fdwxact = (FdwXactState) lfirst(lc); + + /* + * Resolve one foreign transaction. ResolveOneFdwXact() releases and + * removes FdwXactState entry after resolution. + */ + StartTransactionCommand(); + ResolveOneFdwXact(fdwxact); + CommitTransactionCommand(); + } + + if (list_length(heldFdwXactEntries) > 0) + last_resolution_time = GetCurrentTimestamp(); + + list_free(heldFdwXactEntries); + heldFdwXactEntries = NIL; +} diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c index 58091f6b52..0a3f4b383f 100644 --- a/src/backend/access/transam/rmgr.c +++ b/src/backend/access/transam/rmgr.c @@ -10,6 +10,7 @@ #include "access/brin_xlog.h" #include "access/clog.h" #include "access/commit_ts.h" +#include "access/fdwxact_xlog.h" #include "access/generic_xlog.h" #include "access/ginxlog.h" #include "access/gistxlog.h" diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index f67d813c56..29980d56ac 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -77,6 +77,7 @@ #include #include "access/commit_ts.h" +#include "access/fdwxact.h" #include "access/htup_details.h" #include "access/subtrans.h" #include "access/transam.h" @@ -846,6 +847,34 @@ TwoPhaseGetGXact(TransactionId xid, bool lock_held) return result; } +/* + * TwoPhaseExists + * Return true if there is a prepared transaction specified by XID + */ +bool +TwoPhaseExists(TransactionId xid) +{ + int i; + bool found = false; + + LWLockAcquire(TwoPhaseStateLock, LW_SHARED); + + for (i = 0; i < TwoPhaseState->numPrepXacts; i++) + { + GlobalTransaction gxact = TwoPhaseState->prepXacts[i]; + + if (gxact->xid == xid) + { + found = true; + break; + } + } + + LWLockRelease(TwoPhaseStateLock); + + return found; +} + /* * TwoPhaseGetDummyBackendId * Get the dummy backend ID for prepared transaction specified by XID @@ -1556,6 +1585,12 @@ FinishPreparedTransaction(const char *gid, bool isCommit) /* Count the prepared xact as committed or aborted */ AtEOXact_PgStat(isCommit, false); + /* + * If the prepared transaction was a part of a distributed transaction + * notify a resolver process to handle it. + */ + FdwXactLaunchResolversForXid(xid); + /* * And now we can clean up any files we may have left. */ diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index 1e00a3a98e..d007c97c75 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -2131,7 +2131,7 @@ CommitTransaction(void) AtEOXact_Parallel(true); /* Call foreign transaction callbacks at pre-commit phase, if any */ - AtEOXact_FdwXact(true, is_parallel_worker); + PreCommit_FdwXact(is_parallel_worker); /* Shut down the deferred-trigger manager */ AfterTriggerEndXact(true); @@ -2286,6 +2286,7 @@ CommitTransaction(void) AtEOXact_PgStat(true, is_parallel_worker); AtEOXact_Snapshot(true, false); AtEOXact_ApplyLauncher(true); + AtEOXact_FdwXact(true, is_parallel_worker); pgstat_report_xact_timestamp(0); CurrentResourceOwner = NULL; @@ -2559,6 +2560,7 @@ PrepareTransaction(void) PostPrepare_Twophase(); /* PREPARE acts the same as COMMIT as far as GUC is concerned */ + AtEOXact_FdwXact(true, false); AtEOXact_GUC(true, 1); AtEOXact_SPI(true); AtEOXact_Enum(); diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 9cbca6392d..837e0f198a 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -24,6 +24,7 @@ #include "access/clog.h" #include "access/commit_ts.h" +#include "access/fdwxact.h" #include "access/heaptoast.h" #include "access/multixact.h" #include "access/rewriteheap.h" @@ -4670,6 +4671,7 @@ InitControlFile(uint64 sysidentifier) ControlFile->max_worker_processes = max_worker_processes; ControlFile->max_wal_senders = max_wal_senders; ControlFile->max_prepared_xacts = max_prepared_xacts; + ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts; ControlFile->max_locks_per_xact = max_locks_per_xact; ControlFile->wal_level = wal_level; ControlFile->wal_log_hints = wal_log_hints; @@ -6460,6 +6462,9 @@ CheckRequiredParameterValues(void) RecoveryRequiresIntParameter("max_prepared_transactions", max_prepared_xacts, ControlFile->max_prepared_xacts); + RecoveryRequiresIntParameter("max_prepared_foreign_transactions", + max_prepared_foreign_xacts, + ControlFile->max_prepared_foreign_xacts); RecoveryRequiresIntParameter("max_locks_per_transaction", max_locks_per_xact, ControlFile->max_locks_per_xact); @@ -7009,14 +7014,15 @@ StartupXLOG(void) restoreTimeLineHistoryFiles(ThisTimeLineID, recoveryTargetTLI); /* - * Before running in recovery, scan pg_twophase and fill in its status to - * be able to work on entries generated by redo. Doing a scan before - * taking any recovery action has the merit to discard any 2PC files that - * are newer than the first record to replay, saving from any conflicts at - * replay. This avoids as well any subsequent scans when doing recovery - * of the on-disk two-phase data. + * Before running in recovery, scan pg_twophase and pg_fdwxacts, and then + * fill in its status to be able to work on entries generated by redo. + * Doing a scan before taking any recovery action has the merit to discard + * any state files that are newer than the first record to replay, saving + * from any conflicts at replay. This avoids as well any subsequent scans + * when doing recovery of the on-disk two-phase or fdwxact data. */ restoreTwoPhaseData(); + RestoreFdwXactData(); lastFullPageWrites = checkPoint.fullPageWrites; @@ -7218,7 +7224,10 @@ StartupXLOG(void) InitRecoveryTransactionEnvironment(); if (wasShutdown) + { oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids); + oldestActiveXID = PrescanFdwXacts(oldestActiveXID); + } else oldestActiveXID = checkPoint.oldestActiveXid; Assert(TransactionIdIsValid(oldestActiveXID)); @@ -7730,11 +7739,13 @@ StartupXLOG(void) } /* - * Pre-scan prepared transactions to find out the range of XIDs present. - * This information is not quite needed yet, but it is positioned here so - * as potential problems are detected before any on-disk change is done. + * Pre-scan prepared transactions and foreign prepared transacftions to find + * out the range of XIDs present. This information is not quite needed yet, + * but it is positioned here so as potential problems are detected before any + * on-disk change is done. */ oldestActiveXID = PrescanPreparedTransactions(NULL, NULL); + oldestActiveXID = PrescanFdwXacts(oldestActiveXID); /* * Allow ordinary WAL segment creation before any exitArchiveRecovery(), @@ -8066,8 +8077,12 @@ StartupXLOG(void) TrimCLOG(); TrimMultiXact(); - /* Reload shared-memory state for prepared transactions */ + /* + * Reload shared-memory state for prepared transactions and foreign + * prepared transactions. + */ RecoverPreparedTransactions(); + RecoverFdwXacts(); /* * Shutdown the recovery environment. This must occur after @@ -9419,6 +9434,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags) /* We deliberately delay 2PC checkpointing as long as possible */ CheckPointTwoPhase(checkPointRedo); + CheckPointFdwXacts(checkPointRedo); } /* @@ -9955,6 +9971,7 @@ XLogReportParameters(void) max_worker_processes != ControlFile->max_worker_processes || max_wal_senders != ControlFile->max_wal_senders || max_prepared_xacts != ControlFile->max_prepared_xacts || + max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts || max_locks_per_xact != ControlFile->max_locks_per_xact || track_commit_timestamp != ControlFile->track_commit_timestamp) { @@ -9974,6 +9991,7 @@ XLogReportParameters(void) xlrec.max_worker_processes = max_worker_processes; xlrec.max_wal_senders = max_wal_senders; xlrec.max_prepared_xacts = max_prepared_xacts; + xlrec.max_prepared_foreign_xacts = max_prepared_foreign_xacts; xlrec.max_locks_per_xact = max_locks_per_xact; xlrec.wal_level = wal_level; xlrec.wal_log_hints = wal_log_hints; @@ -9992,6 +10010,7 @@ XLogReportParameters(void) ControlFile->max_worker_processes = max_worker_processes; ControlFile->max_wal_senders = max_wal_senders; ControlFile->max_prepared_xacts = max_prepared_xacts; + ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts; ControlFile->max_locks_per_xact = max_locks_per_xact; ControlFile->wal_level = wal_level; ControlFile->wal_log_hints = wal_log_hints; @@ -10198,6 +10217,7 @@ xlog_redo(XLogReaderState *record) RunningTransactionsData running; oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids); + oldestActiveXID = PrescanFdwXacts(oldestActiveXID); /* * Construct a RunningTransactions snapshot representing a shut @@ -10401,6 +10421,7 @@ xlog_redo(XLogReaderState *record) ControlFile->max_worker_processes = xlrec.max_worker_processes; ControlFile->max_wal_senders = xlrec.max_wal_senders; ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts; + ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts; ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact; ControlFile->wal_level = xlrec.wal_level; ControlFile->wal_log_hints = xlrec.wal_log_hints; diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c index 0c37fc1d53..b951c03d20 100644 --- a/src/backend/catalog/dependency.c +++ b/src/backend/catalog/dependency.c @@ -1470,6 +1470,10 @@ doDeletion(const ObjectAddress *object, int flags) RemovePublicationRelById(object->objectId); break; + case OCLASS_USER_MAPPING: + RemoveUserMappingById(object->objectId); + break; + case OCLASS_CAST: case OCLASS_COLLATION: case OCLASS_CONVERSION: @@ -1485,7 +1489,6 @@ doDeletion(const ObjectAddress *object, int flags) case OCLASS_TSTEMPLATE: case OCLASS_FDW: case OCLASS_FOREIGN_SERVER: - case OCLASS_USER_MAPPING: case OCLASS_DEFACL: case OCLASS_EVENT_TRIGGER: case OCLASS_PUBLICATION: diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 999d984068..f36f05f5cc 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -402,6 +402,9 @@ CREATE VIEW pg_prepared_xacts AS CREATE VIEW pg_prepared_statements AS SELECT * FROM pg_prepared_statement() AS P; +CREATE VIEW pg_foreign_xacts AS + SELECT * FROM pg_foreign_xacts() AS F; + CREATE VIEW pg_seclabels AS SELECT l.objoid, l.classoid, l.objsubid, diff --git a/src/backend/commands/dbcommands.c b/src/backend/commands/dbcommands.c index 2b159b60eb..25ad64639f 100644 --- a/src/backend/commands/dbcommands.c +++ b/src/backend/commands/dbcommands.c @@ -23,6 +23,7 @@ #include #include +#include "access/fdwxact.h" #include "access/genam.h" #include "access/heapam.h" #include "access/htup_details.h" @@ -821,6 +822,7 @@ dropdb(const char *dbname, bool missing_ok, bool force) int nslots, nslots_active; int nsubscriptions; + int nfdwxacts; /* * Look up the target database's OID, and get exclusive lock on it. We @@ -910,6 +912,18 @@ dropdb(const char *dbname, bool missing_ok, bool force) "There are %d subscriptions.", nsubscriptions, nsubscriptions))); + /* + * Also check if there is foreign transaction associated with the target + * database. + */ + if ((nfdwxacts = CountFdwXactsForDB(db_id)) > 0) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_IN_USE), + errmsg("database \"%s\" is being used by foreign transaction resolution", + dbname), + errdetail_plural("There is %d foreign transaction.", + "There are %d foreign transactions.", + nfdwxacts, nfdwxacts))); /* * Attempt to terminate all existing connections to the target database if diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index bc36311d38..533d5d59a1 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -13,6 +13,8 @@ */ #include "postgres.h" +#include "access/fdwxact.h" +#include "access/fdwxact_launcher.h" #include "access/htup_details.h" #include "access/reloptions.h" #include "access/table.h" @@ -1060,6 +1062,49 @@ AlterForeignServer(AlterForeignServerStmt *stmt) return address; } +/* + * Drop foreign server + */ +Oid +RemoveForeignServer(DropForeignServerStmt *stmt) +{ + Oid serverid; + ObjectAddress object; + + serverid = GetSysCacheOid1(FOREIGNSERVERNAME, Anum_pg_foreign_server_oid, + CStringGetDatum(stmt->servername)); + + if (!OidIsValid(serverid)) + { + if (!stmt->missing_ok) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("server \"%s\" does not exist", stmt->servername))); + + ereport(NOTICE, + errmsg("server \"%s\" does not exist, skipping", + stmt->servername)); + return InvalidOid; + } + + /* + * Only owner or a superuser can ALTER a SERVER. + */ + if (!pg_foreign_server_ownercheck(serverid, GetUserId())) + aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_FOREIGN_SERVER, + stmt->servername); + + /* Do the deletion */ + object.classId = ForeignServerRelationId; + object.objectId = serverid; + object.objectSubId = 0; + performDeletion(&object, stmt->behavior, 0); + + /* Stop the foreign transaction resolver immediately */ + StopFdwXactResolver(MyDatabaseId, serverid); + + return serverid; +} /* * Common routine to check permission for user-mapping-related DDL @@ -1307,6 +1352,37 @@ AlterUserMapping(AlterUserMappingStmt *stmt) return address; } +/* + * Drop the given user mapping + */ +void +RemoveUserMappingById(Oid umid) +{ + HeapTuple tp; + Relation rel; + + rel = table_open(UserMappingRelationId, RowExclusiveLock); + + tp = SearchSysCache1(USERMAPPINGOID, ObjectIdGetDatum(umid)); + + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for user mapping %u", umid); + + /* + * We cannot drop the user mapping if there is a foreign prepared + * transaction with this user mapping. + */ + if (CountFdwXactsForUserMapping(umid) > 0) + ereport(ERROR, + (errmsg("user mapping %u has unresolved prepared transaction", + umid))); + + CatalogTupleDelete(rel, &tp->t_self); + + ReleaseSysCache(tp); + + table_close(rel, RowExclusiveLock); +} /* * Drop user mapping @@ -1374,6 +1450,7 @@ RemoveUserMapping(DropUserMappingStmt *stmt) user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername); + /* * Do the deletion */ @@ -1386,7 +1463,6 @@ RemoveUserMapping(DropUserMappingStmt *stmt) return umId; } - /* * Create a foreign table * call after DefineRelation(). diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c index f8eb4fa215..6ce76b2aec 100644 --- a/src/backend/foreign/foreign.c +++ b/src/backend/foreign/foreign.c @@ -332,6 +332,12 @@ GetFdwRoutine(Oid fdwhandler) Assert((routine->CommitForeignTransaction && routine->RollbackForeignTransaction) || (!routine->CommitForeignTransaction && !routine->RollbackForeignTransaction)); + /* FDW supporting prepare API must support also commit and rollback APIs */ + Assert((routine->PrepareForeignTransaction && + routine->CommitForeignTransaction && + routine->RollbackForeignTransaction) || + !routine->PrepareForeignTransaction); + return routine; } diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index bd87f23784..aad42485f5 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -4355,6 +4355,18 @@ _copyAlterForeignServerStmt(const AlterForeignServerStmt *from) return newnode; } +static DropForeignServerStmt * +_copyDropForeignServerStmt(const DropForeignServerStmt *from) +{ + DropForeignServerStmt *newnode = makeNode(DropForeignServerStmt); + + COPY_STRING_FIELD(servername); + COPY_SCALAR_FIELD(missing_ok); + COPY_SCALAR_FIELD(behavior); + + return newnode; +} + static CreateUserMappingStmt * _copyCreateUserMappingStmt(const CreateUserMappingStmt *from) { @@ -5604,6 +5616,9 @@ copyObjectImpl(const void *from) case T_AlterForeignServerStmt: retval = _copyAlterForeignServerStmt(from); break; + case T_DropForeignServerStmt: + retval = _copyDropForeignServerStmt(from); + break; case T_CreateUserMappingStmt: retval = _copyCreateUserMappingStmt(from); break; diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index dba3e6b31e..e0b24d1147 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -1977,6 +1977,17 @@ _equalAlterForeignServerStmt(const AlterForeignServerStmt *a, const AlterForeign return true; } +static bool +_equalDropForeignServerStmt(const DropForeignServerStmt *a, + const DropForeignServerStmt *b) +{ + COMPARE_STRING_FIELD(servername); + COMPARE_SCALAR_FIELD(missing_ok); + COMPARE_SCALAR_FIELD(behavior); + + return true; +} + static bool _equalCreateUserMappingStmt(const CreateUserMappingStmt *a, const CreateUserMappingStmt *b) { @@ -3595,6 +3606,9 @@ equal(const void *a, const void *b) case T_AlterForeignServerStmt: retval = _equalAlterForeignServerStmt(a, b); break; + case T_DropForeignServerStmt: + retval = _equalDropForeignServerStmt(a, b); + break; case T_CreateUserMappingStmt: retval = _equalCreateUserMappingStmt(a, b); break; diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index eb24195438..85feb0a600 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -281,7 +281,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); CreateAssertionStmt CreateTransformStmt CreateTrigStmt CreateEventTrigStmt CreateUserStmt CreateUserMappingStmt CreateRoleStmt CreatePolicyStmt CreatedbStmt DeclareCursorStmt DefineStmt DeleteStmt DiscardStmt DoStmt - DropOpClassStmt DropOpFamilyStmt DropStmt + DropOpClassStmt DropOpFamilyStmt DropForeignServerStmt DropStmt DropCastStmt DropRoleStmt DropdbStmt DropTableSpaceStmt DropTransformStmt @@ -980,6 +980,7 @@ stmt: | DiscardStmt | DoStmt | DropCastStmt + | DropForeignServerStmt | DropOpClassStmt | DropOpFamilyStmt | DropOwnedStmt @@ -6339,6 +6340,7 @@ object_type_name: drop_type_name { $$ = $1; } | DATABASE { $$ = OBJECT_DATABASE; } | ROLE { $$ = OBJECT_ROLE; } + | SERVER { $$ = OBJECT_FOREIGN_SERVER; } | SUBSCRIPTION { $$ = OBJECT_SUBSCRIPTION; } | TABLESPACE { $$ = OBJECT_TABLESPACE; } ; @@ -6351,7 +6353,6 @@ drop_type_name: | opt_procedural LANGUAGE { $$ = OBJECT_LANGUAGE; } | PUBLICATION { $$ = OBJECT_PUBLICATION; } | SCHEMA { $$ = OBJECT_SCHEMA; } - | SERVER { $$ = OBJECT_FOREIGN_SERVER; } ; /* object types attached to a table */ @@ -9798,6 +9799,30 @@ DropSubscriptionStmt: DROP SUBSCRIPTION name opt_drop_behavior } ; +/***************************************************************************** + * + * DROP SERVER [ IF EXISTS ] name + * + *****************************************************************************/ + +DropForeignServerStmt: DROP SERVER name opt_drop_behavior + { + DropForeignServerStmt *n = makeNode(DropForeignServerStmt); + n->servername = $3; + n->missing_ok = false; + n->behavior = $4; + $$ = (Node *) n; + } + | DROP SERVER IF_P EXISTS name opt_drop_behavior + { + DropForeignServerStmt *n = makeNode(DropForeignServerStmt); + n->servername = $5; + n->missing_ok = true; + n->behavior = $6; + $$ = (Node *) n; + } + ; + /***************************************************************************** * * QUERY: Define Rewrite Rule diff --git a/src/backend/postmaster/bgworker.c b/src/backend/postmaster/bgworker.c index c40410d73e..89d0219cad 100644 --- a/src/backend/postmaster/bgworker.c +++ b/src/backend/postmaster/bgworker.c @@ -13,6 +13,8 @@ #include "postgres.h" #include "access/parallel.h" +#include "access/fdwxact_resolver.h" +#include "access/fdwxact_launcher.h" #include "libpq/pqsignal.h" #include "miscadmin.h" #include "pgstat.h" @@ -128,6 +130,12 @@ static const struct }, { "ApplyWorkerMain", ApplyWorkerMain + }, + { + "FdwXactResolverMain", FdwXactResolverMain + }, + { + "FdwXactLauncherMain", FdwXactLauncherMain } }; diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index 5a050898fe..d5afe2382c 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -93,6 +93,7 @@ #include #endif +#include "access/fdwxact_launcher.h" #include "access/transam.h" #include "access/xlog.h" #include "bootstrap/bootstrap.h" @@ -925,6 +926,9 @@ PostmasterMain(int argc, char *argv[]) if (max_wal_senders > 0 && wal_level == WAL_LEVEL_MINIMAL) ereport(ERROR, (errmsg("WAL streaming (max_wal_senders > 0) requires wal_level \"replica\" or \"logical\""))); + if (max_prepared_foreign_xacts > 0 && max_foreign_xact_resolvers <= 0) + ereport(ERROR, + (errmsg("preparing foreign transactions (max_prepared_foreign_transactions > 0) requires max_foreign_transaction_resolvers > 0"))); /* * Other one-time internal sanity checks can go here, if they are fast. @@ -990,12 +994,13 @@ PostmasterMain(int argc, char *argv[]) LocalProcessControlFile(false); /* - * Register the apply launcher. Since it registers a background worker, - * it needs to be called before InitializeMaxBackends(), and it's probably - * a good idea to call it before any modules had chance to take the - * background worker slots. + * Register the apply launcher and foreign transaction launcher. Since + * it registers a background worker, it needs to be called before + * InitializeMaxBackends(), and it's probably a good idea to call it + * before any modules had chance to take the background worker slots. */ ApplyLauncherRegister(); + FdwXactLauncherRegister(); /* * process any libraries that should be preloaded at postmaster start diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c index 453efc51e1..eac334c5f6 100644 --- a/src/backend/replication/logical/decode.c +++ b/src/backend/replication/logical/decode.c @@ -179,6 +179,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor case RM_COMMIT_TS_ID: case RM_REPLORIGIN_ID: case RM_GENERIC_ID: + case RM_FDWXACT_ID: /* just deal with xid, and done */ ReorderBufferProcessXid(ctx->reorder, XLogRecGetXid(record), buf.origptr); diff --git a/src/backend/replication/syncrep.c b/src/backend/replication/syncrep.c index bdbc9ef844..c4f4ca8c65 100644 --- a/src/backend/replication/syncrep.c +++ b/src/backend/replication/syncrep.c @@ -143,13 +143,16 @@ static bool SyncRepQueueIsOrderedByLSN(int mode); * represents a commit record. If it doesn't, then we wait only for the WAL * to be flushed if synchronous_commit is set to the higher level of * remote_apply, because only commit records provide apply feedback. + * + * Return true if the waits is cancaled by an interruption. */ -void +bool SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) { char *new_status = NULL; const char *old_status; int mode; + bool canceled = false; /* * This should be called while holding interrupts during a transaction @@ -173,7 +176,7 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) */ if (!SyncRepRequested() || !((volatile WalSndCtlData *) WalSndCtl)->sync_standbys_defined) - return; + return false; /* Cap the level for anything other than commit to remote flush only. */ if (commit) @@ -199,7 +202,7 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) lsn <= WalSndCtl->lsn[mode]) { LWLockRelease(SyncRepLock); - return; + return false; } /* @@ -269,6 +272,7 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) errdetail("The transaction has already committed locally, but might not have been replicated to the standby."))); whereToSendOutput = DestNone; SyncRepCancelWait(); + canceled = true; break; } @@ -285,6 +289,7 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) (errmsg("canceling wait for synchronous replication due to user request"), errdetail("The transaction has already committed locally, but might not have been replicated to the standby."))); SyncRepCancelWait(); + canceled = true; break; } @@ -304,6 +309,7 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) ProcDiePending = true; whereToSendOutput = DestNone; SyncRepCancelWait(); + canceled = true; break; } } @@ -327,6 +333,8 @@ SyncRepWaitForLSN(XLogRecPtr lsn, bool commit) set_ps_display(new_status); pfree(new_status); } + + return canceled; } /* diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index 3e4ec53a97..47179c37a4 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -16,6 +16,8 @@ #include "access/clog.h" #include "access/commit_ts.h" +#include "access/fdwxact.h" +#include "access/fdwxact_launcher.h" #include "access/heapam.h" #include "access/multixact.h" #include "access/nbtree.h" @@ -150,6 +152,8 @@ CreateSharedMemoryAndSemaphores(void) size = add_size(size, BTreeShmemSize()); size = add_size(size, SyncScanShmemSize()); size = add_size(size, AsyncShmemSize()); + size = add_size(size, FdwXactShmemSize()); + size = add_size(size, FdwXactLauncherShmemSize()); #ifdef EXEC_BACKEND size = add_size(size, ShmemBackendArraySize()); #endif @@ -269,6 +273,8 @@ CreateSharedMemoryAndSemaphores(void) BTreeShmemInit(); SyncScanShmemInit(); AsyncShmemInit(); + FdwXactShmemInit(); + FdwXactLauncherShmemInit(); #ifdef EXEC_BACKEND diff --git a/src/backend/storage/ipc/procarray.c b/src/backend/storage/ipc/procarray.c index 793df973b4..417355f795 100644 --- a/src/backend/storage/ipc/procarray.c +++ b/src/backend/storage/ipc/procarray.c @@ -96,6 +96,8 @@ typedef struct ProcArrayStruct TransactionId replication_slot_xmin; /* oldest catalog xmin of any replication slot */ TransactionId replication_slot_catalog_xmin; + /* local transaction id of oldest unresolved distributed transaction */ + TransactionId fdwxact_unresolved_xmin; /* indexes into allProcs[], has PROCARRAY_MAXPROCS entries */ int pgprocnos[FLEXIBLE_ARRAY_MEMBER]; @@ -187,11 +189,13 @@ typedef struct ComputeXidHorizonsResult FullTransactionId latest_completed; /* - * The same for procArray->replication_slot_xmin and. - * procArray->replication_slot_catalog_xmin. + * The same for procArray->replication_slot_xmin, + * procArray->replication_slot_catalog_xmin, and + * procArray->fdwxact_unresolved_xmin. */ TransactionId slot_xmin; TransactionId slot_catalog_xmin; + TransactionId fdwxact_unresolved_xmin; /* * Oldest xid that any backend might still consider running. This needs to @@ -210,8 +214,9 @@ typedef struct ComputeXidHorizonsResult * Oldest xid for which deleted tuples need to be retained in shared * tables. * - * This includes the effects of replication slots. If that's not desired, - * look at shared_oldest_nonremovable_raw; + * This includes the effects of replication slots as unresolved + * foreign transactions. If that's not desired, look at + * shared_oldest_nonremovable_raw; */ TransactionId shared_oldest_nonremovable; @@ -418,6 +423,7 @@ CreateSharedProcArray(void) procArray->lastOverflowedXid = InvalidTransactionId; procArray->replication_slot_xmin = InvalidTransactionId; procArray->replication_slot_catalog_xmin = InvalidTransactionId; + procArray->fdwxact_unresolved_xmin = InvalidTransactionId; ShmemVariableCache->xactCompletionCount = 1; } @@ -1741,6 +1747,7 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h) */ h->slot_xmin = procArray->replication_slot_xmin; h->slot_catalog_xmin = procArray->replication_slot_catalog_xmin; + h->fdwxact_unresolved_xmin = procArray->fdwxact_unresolved_xmin; for (int index = 0; index < arrayP->numProcs; index++) { @@ -1888,6 +1895,15 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h) h->data_oldest_nonremovable = TransactionIdOlder(h->data_oldest_nonremovable, h->slot_xmin); + /* + * Check whether there are unresolved distributed transaction requiring + * an older xmin. + */ + h->shared_oldest_nonremovable = + TransactionIdOlder(h->shared_oldest_nonremovable, h->fdwxact_unresolved_xmin); + h->data_oldest_nonremovable = + TransactionIdOlder(h->data_oldest_nonremovable, h->fdwxact_unresolved_xmin); + /* * The only difference between catalog / data horizons is that the slot's * catalog xmin is applied to the catalog one (so catalogs can be accessed @@ -1947,6 +1963,9 @@ ComputeXidHorizons(ComputeXidHorizonsResult *h) Assert(!TransactionIdIsValid(h->slot_catalog_xmin) || TransactionIdPrecedesOrEquals(h->oldest_considered_running, h->slot_catalog_xmin)); + Assert(!TransactionIdIsValid(h->fdwxact_unresolved_xmin) || + TransactionIdPrecedesOrEquals(h->oldest_considered_running, + h->fdwxact_unresolved_xmin)); /* update approximate horizons with the computed horizons */ GlobalVisUpdateApply(h); @@ -3859,6 +3878,21 @@ ProcArrayGetReplicationSlotXmin(TransactionId *xmin, LWLockRelease(ProcArrayLock); } +/* + * ProcArraySetFdwXactUnresolvedXmin + * + * Install limits to future computations of the xmin horizon to prevent + * vacuum clog from affected transactions needed by resolving distributed + * transaction. + */ +void +ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin) +{ + LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); + procArray->fdwxact_unresolved_xmin = xmin; + LWLockRelease(ProcArrayLock); +} + /* * XidCacheRemoveRunningXids * diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index 6c7cf6c295..a297c746cd 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -53,3 +53,5 @@ XactTruncationLock 44 # 45 was XactTruncationLock until removal of BackendRandomLock WrapLimitsVacuumLock 46 NotifyQueueTailLock 47 +FdwXactLock 48 +FdwXactResolverLock 49 diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 8cea10c901..929061a6bb 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -36,6 +36,8 @@ #include "rusagestub.h" #endif +#include "access/fdwxact_launcher.h" +#include "access/fdwxact_resolver.h" #include "access/parallel.h" #include "access/printtup.h" #include "access/xact.h" @@ -3164,6 +3166,20 @@ ProcessInterrupts(void) */ proc_exit(1); } + else if (IsFdwXactResolver()) + ereport(FATAL, + (errcode(ERRCODE_ADMIN_SHUTDOWN), + errmsg("terminating foreign transaction resolver due to administrator command"))); + else if (IsFdwXactLauncher()) + { + ereport(DEBUG1, + (errmsg_internal("foreign transaction launcher shutting down"))); + /* + * The foreign transaction launcher can be stopped at any time. + * Use exit status 1 so the background worker is restarted. + */ + proc_exit(1); + } else if (RecoveryConflictPending && RecoveryConflictRetryable) { pgstat_report_recovery_conflict(RecoveryConflictReason); diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 7a2da9dab4..95cf27f7cf 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -197,6 +197,7 @@ ClassifyUtilityCommandAsReadOnly(Node *parsetree) case T_CreateUserMappingStmt: case T_CreatedbStmt: case T_DefineStmt: + case T_DropForeignServerStmt: case T_DropOwnedStmt: case T_DropRoleStmt: case T_DropStmt: @@ -1567,6 +1568,10 @@ ProcessUtilitySlow(ParseState *pstate, address = AlterForeignServer((AlterForeignServerStmt *) parsetree); break; + case T_DropForeignServerStmt: + RemoveForeignServer((DropForeignServerStmt *) parsetree); + break; + case T_CreateUserMappingStmt: address = CreateUserMapping((CreateUserMappingStmt *) parsetree); break; @@ -2473,6 +2478,10 @@ CreateCommandTag(Node *parsetree) tag = CMDTAG_ALTER_SERVER; break; + case T_DropForeignServerStmt: + tag = CMDTAG_DROP_SERVER; + break; + case T_CreateUserMappingStmt: tag = CMDTAG_CREATE_USER_MAPPING; break; @@ -2577,9 +2586,6 @@ CreateCommandTag(Node *parsetree) case OBJECT_FDW: tag = CMDTAG_DROP_FOREIGN_DATA_WRAPPER; break; - case OBJECT_FOREIGN_SERVER: - tag = CMDTAG_DROP_SERVER; - break; case OBJECT_OPCLASS: tag = CMDTAG_DROP_OPERATOR_CLASS; break; diff --git a/src/backend/utils/activity/wait_event.c b/src/backend/utils/activity/wait_event.c index 6baf67740c..429a50c591 100644 --- a/src/backend/utils/activity/wait_event.c +++ b/src/backend/utils/activity/wait_event.c @@ -726,6 +726,21 @@ pgstat_get_wait_io(WaitEventIO w) case WAIT_EVENT_LOGICAL_SUBXACT_WRITE: event_name = "LogicalSubxactWrite"; break; + case WAIT_EVENT_FDWXACT_FILE_READ: + event_name = "FdwXactFileRead"; + break; + case WAIT_EVENT_FDWXACT_FILE_SYNC: + event_name = "FdwXactFileSync"; + break; + case WAIT_EVENT_FDWXACT_FILE_WRITE: + event_name = "FdwXactFileWrite"; + break; + case WAIT_EVENT_FDWXACT_LAUNCHER_MAIN: + event_name = "FdwXactLauncherMain"; + break; + case WAIT_EVENT_FDWXACT_RESOLVER_MAIN: + event_name = "FdwXactResolverMain"; + break; /* no default case, so that compiler will warn */ } diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 480e8cd199..65815ec047 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -33,6 +33,7 @@ #include #include "access/commit_ts.h" +#include "access/fdwxact.h" #include "access/gin.h" #include "access/rmgr.h" #include "access/tableam.h" @@ -802,6 +803,10 @@ const char *const config_group_names[] = gettext_noop("Client Connection Defaults / Other Defaults"), /* LOCK_MANAGEMENT */ gettext_noop("Lock Management"), + /* FOREIGN_TRANSACTION */ + gettext_noop("Foreign Transaction"), + /* FOREIGN_TRANSACTION_RESOLVER */ + gettext_noop("Foreign Transaction / Resolver"), /* COMPAT_OPTIONS_PREVIOUS */ gettext_noop("Version and Platform Compatibility / Previous PostgreSQL Versions"), /* COMPAT_OPTIONS_CLIENT */ @@ -2538,6 +2543,49 @@ static struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM, + gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."), + NULL + }, + &max_prepared_foreign_xacts, + 0, 0, INT_MAX, + NULL, NULL, NULL + }, + + { + {"foreign_transaction_resolver_timeout", PGC_SIGHUP, FOREIGN_TRANSACTION_RESOLVER, + gettext_noop("Sets the maximum time to wait for foreign transaction resolution."), + NULL, + GUC_UNIT_MS + }, + &foreign_xact_resolver_timeout, + 60 * 1000, 0, INT_MAX, + NULL, NULL, NULL + }, + + { + {"max_foreign_transaction_resolvers", PGC_POSTMASTER, RESOURCES_MEM, + gettext_noop("Maximum number of foreign transaction resolution processes."), + NULL + }, + &max_foreign_xact_resolvers, + 0, 0, INT_MAX, + NULL, NULL, NULL + }, + + { + {"foreign_transaction_resolution_retry_interval", PGC_SIGHUP, FOREIGN_TRANSACTION_RESOLVER, + gettext_noop("Sets the time to wait before retrying to resolve foreign transaction " + "after a failed attempt."), + NULL, + GUC_UNIT_MS + }, + &foreign_xact_resolution_retry_interval, + 5000, 1, INT_MAX, + NULL, NULL, NULL + }, + #ifdef LOCK_DEBUG { {"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index b696abfe54..9cc35c7109 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -132,6 +132,8 @@ #temp_buffers = 8MB # min 800kB #max_prepared_transactions = 0 # zero disables the feature # (change requires restart) +#max_prepared_foreign_transactions = 0 # zero disables the feature + # (change requires restart) # Caution: it is not advisable to set max_prepared_transactions nonzero unless # you actively intend to use prepared transactions. #work_mem = 4MB # min 64kB @@ -744,6 +746,18 @@ #max_pred_locks_per_page = 2 # min 0 +#------------------------------------------------------------------------------ +# FOREIGN TRANSACTION +#------------------------------------------------------------------------------ + +#max_foreign_transaction_resolvers = 0 # max number of resolver process + # (change requires restart) +#foreign_transaction_resolver_timeout = 60s # in milliseconds; 0 disables +#foreign_transaction_resolution_retry_interval = 5s # time to wait before + # retrying to resolve + # foreign transactions + # after a failed attempt + #------------------------------------------------------------------------------ # VERSION AND PLATFORM COMPATIBILITY #------------------------------------------------------------------------------ diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 152d21e88b..735e4084b3 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -207,6 +207,7 @@ static const char *const subdirs[] = { "pg_wal/archive_status", "pg_commit_ts", "pg_dynshmem", + "pg_fdwxact", "pg_notify", "pg_serial", "pg_snapshots", diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c index f911f98d94..49d47c2ee7 100644 --- a/src/bin/pg_controldata/pg_controldata.c +++ b/src/bin/pg_controldata/pg_controldata.c @@ -296,6 +296,8 @@ main(int argc, char *argv[]) ControlFile->max_wal_senders); printf(_("max_prepared_xacts setting: %d\n"), ControlFile->max_prepared_xacts); + printf(_("max_prepared_foreign_transactions setting: %d\n"), + ControlFile->max_prepared_foreign_xacts); printf(_("max_locks_per_xact setting: %d\n"), ControlFile->max_locks_per_xact); printf(_("track_commit_timestamp setting: %s\n"), diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c index 805dafef07..dd70a0f8a2 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c @@ -710,6 +710,7 @@ GuessControlValues(void) ControlFile.max_wal_senders = 10; ControlFile.max_worker_processes = 8; ControlFile.max_prepared_xacts = 0; + ControlFile.max_prepared_foreign_xacts = 0; ControlFile.max_locks_per_xact = 64; ControlFile.maxAlign = MAXIMUM_ALIGNOF; @@ -914,6 +915,7 @@ RewriteControlFile(void) ControlFile.max_wal_senders = 10; ControlFile.max_worker_processes = 8; ControlFile.max_prepared_xacts = 0; + ControlFile.max_prepared_foreign_xacts = 0; ControlFile.max_locks_per_xact = 64; /* The control file gets flushed here. */ diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c index 852d8ca4b1..b616cea347 100644 --- a/src/bin/pg_waldump/rmgrdesc.c +++ b/src/bin/pg_waldump/rmgrdesc.c @@ -11,6 +11,7 @@ #include "access/brin_xlog.h" #include "access/clog.h" #include "access/commit_ts.h" +#include "access/fdwxact_xlog.h" #include "access/generic_xlog.h" #include "access/ginxlog.h" #include "access/gistxlog.h" diff --git a/src/include/access/fdwxact.h b/src/include/access/fdwxact.h index 1d4a285c75..85854864b9 100644 --- a/src/include/access/fdwxact.h +++ b/src/include/access/fdwxact.h @@ -1,7 +1,7 @@ /* * fdwxact.h * - * PostgreSQL global transaction manager + * PostgreSQL foreign transaction manager definitions * * Portions Copyright (c) 2021, PostgreSQL Global Development Group * @@ -11,13 +11,80 @@ #define FDWXACT_H #include "access/xact.h" +#include "access/fdwxact_xlog.h" #include "foreign/foreign.h" +#include "storage/proc.h" +#include "storage/shmem.h" +#include "storage/s_lock.h" /* Flag passed to FDW transaction management APIs */ #define FDWXACT_FLAG_ONEPHASE 0x01 /* transaction can commit/rollback * without preparation */ #define FDWXACT_FLAG_PARALLEL_WORKER 0x02 /* is parallel worker? */ +/* Enum to track the status of foreign transaction */ +typedef enum +{ + FDWXACT_STATUS_INVALID = 0, + FDWXACT_STATUS_PREPARING, /* foreign transaction is being prepared */ + FDWXACT_STATUS_PREPARED, /* foreign transaction is prepared */ + FDWXACT_STATUS_COMMITTING, /* foreign prepared transaction is committed */ + FDWXACT_STATUS_ABORTING /* foreign prepared transaction is aborted */ +} FdwXactStatus; + +/* + * Shared memory state of a single foreign transaction. + */ +typedef struct FdwXactStateData *FdwXactState; +typedef struct FdwXactStateData +{ + FdwXactState fdwxact_free_next; /* Next free FdwXactState entry */ + + /* Information relevant with foreign transaction */ + FdwXactStateOnDiskData data; + + /* Foreign transaction status */ + FdwXactStatus status; + slock_t mutex; /* protect the above field */ + + /* + * Note that we need to keep track of two LSNs for each FdwXactState. We keep + * track of the start LSN because this is the address we must use to read + * state data back from WAL when committing a FdwXactState. We keep track of + * the end LSN because that is the LSN we need to wait for prior to + * commit. + */ + XLogRecPtr insert_start_lsn; /* XLOG offset of inserting this entry + * start */ + XLogRecPtr insert_end_lsn; /* XLOG offset of inserting this entry end */ + + bool valid; /* has the entry been complete and written to + * file? */ + BackendId locking_backend; /* backend currently working on the fdw xact */ + bool ondisk; /* true if prepare state file is on disk */ + bool inredo; /* true if entry was added via xlog_redo */ +} FdwXactStateData; + +/* + * Shared memory layout for maintaining foreign prepared transaction entries. + * Adding or removing FdwXactState entry needs to hold FdwXactLock in exclusive mode, + * and iterating fdwXacts needs that in shared mode. + */ +typedef struct +{ + /* Head of linked list of free FdwXactStateData structs */ + FdwXactState free_fdwxacts; + + /* Number of valid foreign transaction entries */ + int num_xacts; + + /* Upto max_prepared_foreign_xacts entries in the array */ + FdwXactState xacts[FLEXIBLE_ARRAY_MEMBER]; /* Variable length array */ +} FdwXactCtlData; + +/* Pointer to the shared memory holding the foreign transactions data */ +FdwXactCtlData *FdwXactCtl; + /* State data for foreign transaction resolution, passed to FDW callbacks */ typedef struct FdwXactInfo { @@ -25,10 +92,29 @@ typedef struct FdwXactInfo UserMapping *usermapping; int flags; /* OR of FDWXACT_FLAG_xx flags */ + char *identifier; } FdwXactInfo; +/* GUC parameters */ +extern int max_prepared_foreign_xacts; +extern int max_foreign_xact_resolvers; +extern int foreign_xact_resolution_retry_interval; +extern int foreign_xact_resolver_timeout; + /* Function declarations */ +extern void PreCommit_FdwXact(bool is_parallel_worker); extern void AtEOXact_FdwXact(bool isCommit, bool is_parallel_worker); +extern Size FdwXactShmemSize(void); +extern void FdwXactShmemInit(void); extern void AtPrepare_FdwXact(void); +extern bool FdwXactIsForeignTwophaseCommitRequired(void); +extern int CountFdwXactsForUserMapping(Oid umid); +extern int CountFdwXactsForDB(Oid dbid); +extern void FdwXactLaunchResolversForXid(TransactionId xid); +extern void CheckPointFdwXacts(XLogRecPtr redo_horizon); +extern void ResolveOneFdwXact(FdwXactState fdwxact); +extern void RestoreFdwXactData(void); +extern void RecoverFdwXacts(void); +extern TransactionId PrescanFdwXacts(TransactionId oldestActiveXid); #endif /* FDWXACT_H */ diff --git a/src/include/access/fdwxact_launcher.h b/src/include/access/fdwxact_launcher.h new file mode 100644 index 0000000000..6381aa6e55 --- /dev/null +++ b/src/include/access/fdwxact_launcher.h @@ -0,0 +1,29 @@ +/*------------------------------------------------------------------------- + * + * fdwxact_launcher.h + * PostgreSQL foreign transaction launcher definitions + * + * Portions Copyright (c) 2021, PostgreSQL Global Development Group + * + * src/include/access/fdwxact_launcher.h + * + *------------------------------------------------------------------------- + */ + +#ifndef FDWXACT_LAUNCHER_H +#define FDWXACT_LAUNCHER_H + +#include "access/fdwxact.h" +#include "access/resolver_internal.h" + +extern void FdwXactLauncherRegister(void); +extern void FdwXactLauncherMain(Datum main_arg); +extern void RequestToLaunchFdwXactResolver(void); +extern void LaunchOrWakeupFdwXactResolver(List *serveroids_orig); +extern Size FdwXactLauncherShmemSize(void); +extern void FdwXactLauncherShmemInit(void); +extern bool IsFdwXactLauncher(void); +extern void StopFdwXactResolver(Oid dbid, Oid serverid); + + +#endif /* FDWXACT_LAUNCHER_H */ diff --git a/src/include/access/fdwxact_resolver.h b/src/include/access/fdwxact_resolver.h new file mode 100644 index 0000000000..9301ada5bb --- /dev/null +++ b/src/include/access/fdwxact_resolver.h @@ -0,0 +1,22 @@ +/*------------------------------------------------------------------------- + * + * fdwxact_resolver.h + * PostgreSQL foreign transaction resolver definitions + * + * Portions Copyright (c) 2021, PostgreSQL Global Development Group + * + * src/include/access/fdwxact_resolver.h + * + *------------------------------------------------------------------------- + */ +#ifndef FDWXACT_RESOLVER_H +#define FDWXACT_RESOLVER_H + +#include "access/fdwxact.h" + +extern void FdwXactResolverMain(Datum main_arg); +extern bool IsFdwXactResolver(void); + +extern int foreign_xact_resolver_timeout; + +#endif /* FDWXACT_RESOLVER_H */ diff --git a/src/include/access/fdwxact_xlog.h b/src/include/access/fdwxact_xlog.h new file mode 100644 index 0000000000..a1a10b71b2 --- /dev/null +++ b/src/include/access/fdwxact_xlog.h @@ -0,0 +1,49 @@ +/*------------------------------------------------------------------------- + * + * fdwxact_xlog.h + * Foreign transaction XLOG definitions. + * + * Portions Copyright (c) 2021, PostgreSQL Global Development Group + * + * src/include/access/fdwxact_xlog.h + * + *------------------------------------------------------------------------- + */ +#ifndef FDWXACT_XLOG_H +#define FDWXACT_XLOG_H + +#include "access/xlogreader.h" +#include "lib/stringinfo.h" + +/* Info types for logs related to FDW transactions */ +#define XLOG_FDWXACT_INSERT 0x00 +#define XLOG_FDWXACT_REMOVE 0x10 + +/* Maximum length of the prepared transaction id, borrowed from twophase.c */ +#define FDWXACT_ID_MAX_LEN 200 + +/* + * On disk file structure, also used to WAL + */ +typedef struct +{ + TransactionId xid; + Oid dbid; + Oid umid; + Oid serverid; + Oid owner; + char identifier[FDWXACT_ID_MAX_LEN]; /* foreign txn prepare id */ +} FdwXactStateOnDiskData; + +typedef struct xl_fdwxact_remove +{ + TransactionId xid; + Oid umid; + bool force; +} xl_fdwxact_remove; + +extern void fdwxact_redo(XLogReaderState *record); +extern void fdwxact_desc(StringInfo buf, XLogReaderState *record); +extern const char *fdwxact_identify(uint8 info); + +#endif /* FDWXACT_XLOG_H */ diff --git a/src/include/access/resolver_internal.h b/src/include/access/resolver_internal.h new file mode 100644 index 0000000000..89fc6a5653 --- /dev/null +++ b/src/include/access/resolver_internal.h @@ -0,0 +1,59 @@ +/*------------------------------------------------------------------------- + * + * resolver_internal.h + * Internal headers shared by fdwxact resolvers. + * + * Portions Copyright (c) 2021, PostgreSQL Global Development Group + * + * src/include/access/resolver_internal.h + * + *------------------------------------------------------------------------- + */ + +#ifndef RESOLVER_INTERNAL_H +#define RESOLVER_INTERNAL_H + +#include "storage/latch.h" +#include "storage/shmem.h" +#include "storage/spin.h" +#include "utils/timestamp.h" + +/* + * Each foreign transaction resolver has a FdwXactResolver struct in + * shared memory. This struct is protected by FdwXactResolverLock. + */ +typedef struct FdwXactResolver +{ + pid_t pid; /* this resolver's PID, or 0 if not active */ + Oid dbid; + Oid serverid; + + /* Indicates if this slot is used of free */ + bool in_use; + + /* Protect shared variables shown above */ + slock_t mutex; + + /* + * Pointer to the resolver's patch. Used by backends to wake up this + * resolver when it has work to do. NULL if the resolver isn't active. + */ + Latch *latch; +} FdwXactResolver; + +/* There is one FdwXactResolverCtlData struct for the whole database cluster */ +typedef struct FdwXactResolverCtlData +{ + /* Supervisor process and latch */ + pid_t launcher_pid; + Latch *launcher_latch; + + FdwXactResolver resolvers[FLEXIBLE_ARRAY_MEMBER]; +} FdwXactResolverCtlData; +#define SizeOfFdwXactResolverCtlData \ + (offsetof(FdwXactResolverCtlData, resolvers) + sizeof(FdwXactResolver)) + +extern FdwXactResolverCtlData *FdwXactResolverCtl; +extern FdwXactResolver *MyFdwXactResolver; + +#endif /* RESOLVER_INTERNAL_H */ diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h index f582cf535f..5ab1f57212 100644 --- a/src/include/access/rmgrlist.h +++ b/src/include/access/rmgrlist.h @@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL) PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask) PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL) +PG_RMGR(RM_FDWXACT_ID, "Fdw Transaction", fdwxact_redo, fdwxact_desc, fdwxact_identify, NULL, NULL, NULL) diff --git a/src/include/access/twophase.h b/src/include/access/twophase.h index 91786da784..3d35f89ae0 100644 --- a/src/include/access/twophase.h +++ b/src/include/access/twophase.h @@ -36,6 +36,7 @@ extern void PostPrepare_Twophase(void); extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid, bool lock_held); extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid, bool lock_held); +extern bool TwoPhaseExists(TransactionId xid); extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid, TimestampTz prepared_at, diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index 3b5eceff65..21fa835687 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -236,6 +236,7 @@ typedef struct xl_parameter_change int max_worker_processes; int max_wal_senders; int max_prepared_xacts; + int max_prepared_foreign_xacts; int max_locks_per_xact; int wal_level; bool wal_log_hints; diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h index e3f48158ce..5673ec7299 100644 --- a/src/include/catalog/pg_control.h +++ b/src/include/catalog/pg_control.h @@ -179,6 +179,7 @@ typedef struct ControlFileData int max_worker_processes; int max_wal_senders; int max_prepared_xacts; + int max_prepared_foreign_xacts; int max_locks_per_xact; bool track_commit_timestamp; diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index fde251fa4f..7569f4cdbd 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -6123,6 +6123,24 @@ proargnames => '{type,object_names,object_args,classid,objid,objsubid}', prosrc => 'pg_get_object_address' }, +{ oid => '9706', descr => 'view foreign transactions', + proname => 'pg_foreign_xacts', prorows => '100', proretset => 't', + provolatile => 'v', prorettype => 'record', proargtypes => '', + proallargtypes => '{xid,oid,oid,oid,text,text,int4}', + proargmodes => '{o,o,o,o,o,o,o}', + proargnames => '{xid,umid,ownerid,dbid,state,identifier,locker_pid}', + prosrc => 'pg_foreign_xacts' }, +{ oid => '9707', descr => 'remove foreign transaction without resolution', + proname => 'pg_remove_foreign_xact', provolatile => 'v', prorettype => 'bool', + proargtypes => 'xid oid', + proargnames => '{xid,umid}', + prosrc => 'pg_remove_foreign_xact' }, +{ oid => '9708', descr => 'resolve one foreign transaction', + proname => 'pg_resolve_foreign_xact', provolatile => 'v', prorettype => 'bool', + proargtypes => 'xid oid', + proargnames => '{xid,umid}', + prosrc => 'pg_resolve_foreign_xact' }, + { oid => '2079', descr => 'is table visible in search path?', proname => 'pg_table_is_visible', procost => '10', provolatile => 's', prorettype => 'bool', proargtypes => 'oid', prosrc => 'pg_table_is_visible' }, @@ -6243,6 +6261,11 @@ proname => 'pg_walfile_name', prorettype => 'text', proargtypes => 'pg_lsn', prosrc => 'pg_walfile_name' }, +{ oid => '9709', + descr => 'stop a running foreign transaction resolver process', + proname => 'pg_stop_foreign_xact_resolver', provolatile => 'v', prorettype => 'bool', + proargtypes => 'text', prosrc => 'pg_stop_foreign_xact_resolver'}, + { oid => '3165', descr => 'difference in bytes, given two wal locations', proname => 'pg_wal_lsn_diff', prorettype => 'numeric', proargtypes => 'pg_lsn pg_lsn', prosrc => 'pg_wal_lsn_diff' }, diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h index 42bf1c7519..a38a451afd 100644 --- a/src/include/commands/defrem.h +++ b/src/include/commands/defrem.h @@ -125,9 +125,11 @@ extern ObjectAddress CreateForeignDataWrapper(CreateFdwStmt *stmt); extern ObjectAddress AlterForeignDataWrapper(AlterFdwStmt *stmt); extern ObjectAddress CreateForeignServer(CreateForeignServerStmt *stmt); extern ObjectAddress AlterForeignServer(AlterForeignServerStmt *stmt); +extern Oid RemoveForeignServer(DropForeignServerStmt *stmt); extern ObjectAddress CreateUserMapping(CreateUserMappingStmt *stmt); extern ObjectAddress AlterUserMapping(AlterUserMappingStmt *stmt); extern Oid RemoveUserMapping(DropUserMappingStmt *stmt); +extern void RemoveUserMappingById(Oid umid); extern void CreateForeignTable(CreateForeignTableStmt *stmt, Oid relid); extern void ImportForeignSchema(ImportForeignSchemaStmt *stmt); extern Datum transformGenericOptions(Oid catalogId, diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h index c3539a4d73..5338f4f2d9 100644 --- a/src/include/foreign/fdwapi.h +++ b/src/include/foreign/fdwapi.h @@ -192,6 +192,7 @@ typedef void (*ForeignAsyncConfigureWait_function) (AsyncRequest *areq); typedef void (*ForeignAsyncNotify_function) (AsyncRequest *areq); +typedef void (*PrepareForeignTransaction_function) (FdwXactInfo *finfo); typedef void (*CommitForeignTransaction_function) (FdwXactInfo *finfo); typedef void (*RollbackForeignTransaction_function) (FdwXactInfo *finfo); @@ -287,6 +288,7 @@ typedef struct FdwRoutine /* Support functions for transaction management */ CommitForeignTransaction_function CommitForeignTransaction; RollbackForeignTransaction_function RollbackForeignTransaction; + PrepareForeignTransaction_function PrepareForeignTransaction; } FdwRoutine; diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index d9e417bcd7..3bea64a44b 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -403,6 +403,7 @@ typedef enum NodeTag T_AlterFdwStmt, T_CreateForeignServerStmt, T_AlterForeignServerStmt, + T_DropForeignServerStmt, T_CreateUserMappingStmt, T_AlterUserMappingStmt, T_DropUserMappingStmt, diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index def9651b34..4ecd453689 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -2387,7 +2387,7 @@ typedef struct AlterFdwStmt } AlterFdwStmt; /* ---------------------- - * Create/Alter FOREIGN SERVER Statements + * Create/Alter/Drop FOREIGN SERVER Statements * ---------------------- */ @@ -2411,6 +2411,14 @@ typedef struct AlterForeignServerStmt bool has_version; /* version specified */ } AlterForeignServerStmt; +typedef struct DropForeignServerStmt +{ + NodeTag type; + char *servername; /* server name */ + bool missing_ok; /* Skip error if missing? */ + DropBehavior behavior; /* RESTRICT or CASCADE behavior */ +} DropForeignServerStmt; + /* ---------------------- * Create FOREIGN TABLE Statement * ---------------------- diff --git a/src/include/replication/syncrep.h b/src/include/replication/syncrep.h index 4266afde8b..476fe8c688 100644 --- a/src/include/replication/syncrep.h +++ b/src/include/replication/syncrep.h @@ -82,7 +82,7 @@ extern char *syncrep_parse_error_msg; extern char *SyncRepStandbyNames; /* called by user backend */ -extern void SyncRepWaitForLSN(XLogRecPtr lsn, bool commit); +extern bool SyncRepWaitForLSN(XLogRecPtr lsn, bool commit); /* called at backend exit */ extern void SyncRepCleanupAtProcExit(void); diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h index b01fa52139..300a4cf5b6 100644 --- a/src/include/storage/procarray.h +++ b/src/include/storage/procarray.h @@ -93,5 +93,6 @@ extern void ProcArraySetReplicationSlotXmin(TransactionId xmin, extern void ProcArrayGetReplicationSlotXmin(TransactionId *xmin, TransactionId *catalog_xmin); +extern void ProcArraySetFdwXactUnresolvedXmin(TransactionId xmin); #endif /* PROCARRAY_H */ diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h index 6b40f1eeb8..35802eac86 100644 --- a/src/include/utils/guc_tables.h +++ b/src/include/utils/guc_tables.h @@ -89,6 +89,8 @@ enum config_group CLIENT_CONN_PRELOAD, CLIENT_CONN_OTHER, LOCK_MANAGEMENT, + FOREIGN_TRANSACTION, + FOREIGN_TRANSACTION_RESOLVER, COMPAT_OPTIONS_PREVIOUS, COMPAT_OPTIONS_CLIENT, ERROR_HANDLING_OPTIONS, diff --git a/src/include/utils/wait_event.h b/src/include/utils/wait_event.h index 6c6ec2e711..915c174d1c 100644 --- a/src/include/utils/wait_event.h +++ b/src/include/utils/wait_event.h @@ -224,7 +224,12 @@ typedef enum WAIT_EVENT_LOGICAL_CHANGES_READ, WAIT_EVENT_LOGICAL_CHANGES_WRITE, WAIT_EVENT_LOGICAL_SUBXACT_READ, - WAIT_EVENT_LOGICAL_SUBXACT_WRITE + WAIT_EVENT_LOGICAL_SUBXACT_WRITE, + WAIT_EVENT_FDWXACT_FILE_READ, + WAIT_EVENT_FDWXACT_FILE_SYNC, + WAIT_EVENT_FDWXACT_FILE_WRITE, + WAIT_EVENT_FDWXACT_LAUNCHER_MAIN, + WAIT_EVENT_FDWXACT_RESOLVER_MAIN } WaitEventIO; diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index e5ab11275d..443a6715df 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1352,6 +1352,14 @@ pg_file_settings| SELECT a.sourcefile, a.applied, a.error FROM pg_show_all_file_settings() a(sourcefile, sourceline, seqno, name, setting, applied, error); +pg_foreign_xacts| SELECT f.xid, + f.umid, + f.ownerid, + f.dbid, + f.state, + f.identifier, + f.locker_pid + FROM pg_foreign_xacts() f(xid, umid, ownerid, dbid, state, identifier, locker_pid); pg_group| SELECT pg_authid.rolname AS groname, pg_authid.oid AS grosysid, ARRAY( SELECT pg_auth_members.member -- 2.24.3 (Apple Git-128)