diff --git a/contrib/fdw_transaction_resovler/Makefile b/contrib/fdw_transaction_resovler/Makefile new file mode 100644 index 0000000..0d2e0e9 --- /dev/null +++ b/contrib/fdw_transaction_resovler/Makefile @@ -0,0 +1,15 @@ +# contrib/fdw_transaction_resolver/Makefile + +MODULES = fdw_transaction_resolver +PGFILEDESC = "fdw_transaction_resolver - foreign transaction resolver demon" + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/fdw_transaction_resolver +top_builddir = ../../ +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/contrib/fdw_transaction_resovler/TAGS b/contrib/fdw_transaction_resovler/TAGS new file mode 120000 index 0000000..cf64c85 --- /dev/null +++ b/contrib/fdw_transaction_resovler/TAGS @@ -0,0 +1 @@ +/home/masahiko/pgsql/source/postgresql/TAGS \ No newline at end of file diff --git a/contrib/fdw_transaction_resovler/fdw_transaction_resolver.c b/contrib/fdw_transaction_resovler/fdw_transaction_resolver.c new file mode 100644 index 0000000..055d458 --- /dev/null +++ b/contrib/fdw_transaction_resovler/fdw_transaction_resolver.c @@ -0,0 +1,457 @@ +/* ------------------------------------------------------------------------- + * + * fdw_transaction_resolver.c + * + * Contrib module to launch foreign transaction resolver to resolve unresolved + * transactions prepared on foreign servers. + * + * The extension launches foreign transaction resolver launcher process as a + * background worker. The launcher then launches separate background worker + * process to resolve the foreign transaction in each database. The worker + * process simply connects to the database specified and calls pg_fdw_xact_resolve() + * function, which tries to resolve the transactions. The launcher process + * launches at most one worker at a time. + * + * Copyright (C) 2017, PostgreSQL Global Development Group + * + * IDENTIFICATION + * contrib/fdw_transaction_resolver/fdw_transaction_resolver.c + * + * ------------------------------------------------------------------------- + */ +#include "postgres.h" + +/* These are always necessary for a bgworker */ +#include "miscadmin.h" +#include "postmaster/bgworker.h" +#include "storage/ipc.h" +#include "storage/latch.h" +#include "storage/lwlock.h" +#include "storage/proc.h" +#include "storage/shmem.h" + +/* these headers are used by this particular worker's code */ +#include "access/heapam.h" +#include "access/htup_details.h" +#include "access/xact.h" +#include "access/fdw_xact.h" +#include "catalog/pg_database.h" +#include "executor/spi.h" +#include "fmgr.h" +#include "lib/stringinfo.h" +#include "pgstat.h" +#include "utils/builtins.h" +#include "utils/snapmgr.h" +#include "utils/timestamp.h" +#include "tcop/utility.h" + +PG_MODULE_MAGIC; + +void _PG_init(void); + +/* + * Flags set by interrupt handlers of foreign transaction resolver for later + * service in the main loop. + */ +static volatile sig_atomic_t got_sighup = false; +static volatile sig_atomic_t got_sigterm = false; +static volatile sig_atomic_t got_sigquit = false; +static volatile sig_atomic_t got_sigusr1 = false; + +static void FDWXactResolver_worker_main(Datum dbid_datum); +static void FDWXactResolverMain(Datum main_arg); +static List *get_database_list(void); + +/* GUC variable */ +static int fx_resolver_naptime; + +/* + * Signal handler for SIGTERM + * Set a flag to let the main loop to terminate, and set our latch to wake + * it up. + */ +static void +FDWXactResolver_SIGTERM(SIGNAL_ARGS) +{ + int save_errno = errno; + + got_sigterm = true; + SetLatch(MyLatch); + + errno = save_errno; +} + +/* + * Signal handler for SIGQUIT + * Set a flag to let the main loop to terminate, and set our latch to wake + * it up. + */ +static void +FDWXactResolver_SIGQUIT(SIGNAL_ARGS) +{ + int save_errno = errno; + + got_sigquit = true; + SetLatch(MyLatch); + + errno = save_errno; +} + +/* + * Signal handler for SIGHUP + * Set a flag to tell the main loop to reread the config file, and set + * our latch to wake it up. + */ +static void +FDWXactResolver_SIGHUP(SIGNAL_ARGS) +{ + int save_errno = errno; + + got_sighup = true; + SetLatch(MyLatch); + + errno = save_errno; +} + +static void +FDWXactResolver_SIGUSR1(SIGNAL_ARGS) +{ + int save_errno = errno; + + got_sigusr1 = true; + SetLatch(MyLatch); + + errno = save_errno; +} + +/* + * Entrypoint of this module. + * + * Launches the foreign transaction resolver demon. + */ +void +_PG_init(void) +{ + BackgroundWorker worker; + + if (!process_shared_preload_libraries_in_progress) + return; + + DefineCustomIntVariable("fdw_transaction_resolver.naptime", + "Time to sleep between fdw_transaction_resolver runs.", + NULL, + &fx_resolver_naptime, + 60, + 1, + INT_MAX, + PGC_SIGHUP, + 0, + NULL, NULL, NULL); + + /* set up common data for all our workers */ + + /* + * For some reason unless background worker set + * BGWORKER_BACKEND_DATABASE_CONNECTION, it's not added to BackendList and + * hence notification to this backend is not enabled. So set that flag + * even if the backend itself doesn't need database connection. + */ + worker.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; + worker.bgw_start_time = BgWorkerStart_RecoveryFinished; + worker.bgw_restart_time = 5; + snprintf(worker.bgw_name, BGW_MAXLEN, "foreign transaction resolver launcher"); + sprintf(worker.bgw_library_name, "fdw_transaction_resolver"); + sprintf(worker.bgw_function_name, "FDWXactResolverMain"); + worker.bgw_main_arg = (Datum) 0; /* Craft some dummy arg. */ + worker.bgw_notify_pid = 0; + + RegisterBackgroundWorker(&worker); +} + +void +FDWXactResolverMain(Datum main_arg) +{ + /* For launching background worker */ + BackgroundWorker worker; + BackgroundWorkerHandle *handle = NULL; + pid_t pid; + List *dbid_list = NIL; + TimestampTz launched_time = GetCurrentTimestamp(); + TimestampTz next_launch_time = launched_time + (fx_resolver_naptime * 1000L); + + ereport(LOG, + (errmsg("fdw_transaction_resolver launcher started"))); + + /* Properly accept or ignore signals the postmaster might send us */ + pqsignal(SIGHUP, FDWXactResolver_SIGHUP); /* set flag to read config + * file */ + pqsignal(SIGTERM, FDWXactResolver_SIGTERM); /* request shutdown */ + pqsignal(SIGQUIT, FDWXactResolver_SIGQUIT); /* hard crash time */ + pqsignal(SIGUSR1, FDWXactResolver_SIGUSR1); + + /* Unblock signals */ + BackgroundWorkerUnblockSignals(); + + /* Initialize connection */ + BackgroundWorkerInitializeConnection(NULL, NULL); + + /* + * Main loop: do this until the SIGTERM handler tells us to terminate + */ + while (!got_sigterm) + { + int rc; + int naptime_msec; + TimestampTz current_time = GetCurrentTimestamp(); + + /* Determine sleep time */ + naptime_msec = (fx_resolver_naptime * 1000L) - (current_time - launched_time); + + if (naptime_msec < 0) + naptime_msec = 0; + + /* + * Background workers mustn't call usleep() or any direct equivalent: + * instead, they may wait on their process latch, which sleeps as + * necessary, but is awakened if postmaster dies. That way the + * background process goes away immediately in an emergency. + */ + rc = WaitLatch(MyLatch, + WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, + naptime_msec, + WAIT_EVENT_PG_SLEEP); + ResetLatch(MyLatch); + + /* emergency bailout if postmaster has died */ + if (rc & WL_POSTMASTER_DEATH) + proc_exit(1); + + /* + * Postmaster wants to stop this process. Exit with non-zero code, so + * that the postmaster starts this process again. The worker processes + * will receive the signal and end themselves. This process will + * restart them if necessary. + */ + if (got_sigquit) + proc_exit(2); + + /* In case of a SIGHUP, just reload the configuration */ + if (got_sighup) + { + got_sighup = false; + ProcessConfigFile(PGC_SIGHUP); + } + + if (got_sigusr1) + { + got_sigusr1 = false; + + /* If we had started a worker check whether it completed */ + if (handle) + { + BgwHandleStatus status; + + status = GetBackgroundWorkerPid(handle, &pid); + if (status == BGWH_STOPPED) + handle = NULL; + } + } + + current_time = GetCurrentTimestamp(); + + /* + * If no background worker is running, we can start one if there are + * unresolved foreign transactions. + */ + if (!handle && + TimestampDifferenceExceeds(next_launch_time, current_time, naptime_msec)) + { + Oid dbid; + + /* Get the database list if empty */ + if (!dbid_list) + dbid_list = get_database_list(); + + /* Launch a worker if dbid_list has database */ + if (dbid_list) + { + /* Work on the first dbid, and remove it from the list */ + dbid = linitial_oid(dbid_list); + dbid_list = list_delete_oid(dbid_list, dbid); + + Assert(OidIsValid(dbid)); + + /* Start the foreign transaction resolver */ + worker.bgw_flags = BGWORKER_SHMEM_ACCESS | + BGWORKER_BACKEND_DATABASE_CONNECTION; + worker.bgw_start_time = BgWorkerStart_RecoveryFinished; + + /* We will start another worker if needed */ + worker.bgw_restart_time = BGW_NEVER_RESTART; + sprintf(worker.bgw_library_name, "fdw_transaction_resolver"); + sprintf(worker.bgw_function_name, "FDWXactResolver_worker_main"); + snprintf(worker.bgw_name, BGW_MAXLEN, "foreign transaction resolver (dbid %u)", dbid); + worker.bgw_main_arg = ObjectIdGetDatum(dbid); + + /* set bgw_notify_pid so that we can wait for it to finish */ + worker.bgw_notify_pid = MyProcPid; + + RegisterDynamicBackgroundWorker(&worker, &handle); + } + + /* Set next launch time */ + launched_time = current_time; + next_launch_time = TimestampTzPlusMilliseconds(launched_time, + fx_resolver_naptime * 1000L); + } + } + + /* Time to exit */ + ereport(LOG, + (errmsg("foreign transaction resolver shutting down"))); + + proc_exit(0); /* done */ +} + +/* FDWXactWorker_SIGTERM + * Terminates the foreign transaction resolver worker process */ +static void +FDWXactWorker_SIGTERM(SIGNAL_ARGS) +{ + /* Just terminate the current process */ + proc_exit(1); +} + +/* Per database foreign transaction resolver */ +static void +FDWXactResolver_worker_main(Datum dbid_datum) +{ + char *command = "SELECT * FROM pg_fdw_xact_resolve() WHERE status = 'resolved'"; + Oid dbid = DatumGetObjectId(dbid_datum); + int ret; + + /* + * This background worker does not loop infinitely, so we need handler + * only for SIGTERM, in which case the process should just exit quickly. + */ + pqsignal(SIGTERM, FDWXactWorker_SIGTERM); + pqsignal(SIGQUIT, FDWXactWorker_SIGTERM); + + /* Unblock signals */ + BackgroundWorkerUnblockSignals(); + + /* + * Run this background worker in superuser mode, so that all the foreign + * server and user information isaccessible. + */ + BackgroundWorkerInitializeConnectionByOid(dbid, InvalidOid); + + /* + * Start a transaction on which we can call resolver function. Note that + * each StartTransactionCommand() call should be preceded by a + * SetCurrentStatementStartTimestamp() call, which sets both the time for + * the statement we're about the run, and also the transaction start time. + * Also, each other query sent to SPI should probably be preceded by + * SetCurrentStatementStartTimestamp(), so that statement start time is + * always up to date. + * + * The SPI_connect() call lets us run queries through the SPI manager, and + * the PushActiveSnapshot() call creates an "active" snapshot which is + * necessary for queries to have MVCC data to work on. + * + * The pgstat_report_activity() call makes our activity visible through + * the pgstat views. + */ + SetCurrentStatementStartTimestamp(); + StartTransactionCommand(); + SPI_connect(); + PushActiveSnapshot(GetTransactionSnapshot()); + pgstat_report_activity(STATE_RUNNING, command); + + /* Run the resolver function */ + ret = SPI_execute(command, false, 0); + + if (ret < 0) + elog(LOG, "error running pg_fdw_xact_resolve() within database %d", + dbid); + + if (SPI_processed > 0) + ereport(LOG, + (errmsg("resolved %lu foreign transactions", SPI_processed))); + + /* + * And finish our transaction. + */ + SPI_finish(); + PopActiveSnapshot(); + CommitTransactionCommand(); + pgstat_report_activity(STATE_IDLE, NULL); + + /* Done exit now */ + proc_exit(0); +} + +/* Get database list */ +static List * +get_database_list(void) +{ + List *dblist = NIL; + ListCell *cell; + ListCell *next; + ListCell *prev = NULL; + HeapScanDesc scan; + HeapTuple tup; + Relation rel; + MemoryContext resultcxt; + + /* This is the context that we will allocate our output data in */ + resultcxt = CurrentMemoryContext; + + SetCurrentStatementStartTimestamp(); + StartTransactionCommand(); + (void) GetTransactionSnapshot(); + + rel = heap_open(DatabaseRelationId, AccessShareLock); + scan = heap_beginscan_catalog(rel, 0, NULL); + + while (HeapTupleIsValid(tup = heap_getnext(scan, ForwardScanDirection))) + { + MemoryContext oldcxt; + + /* + * Allocate our results in the caller's context, not the + * transaction's. We do this inside the loop, and restore the original + * context at the end, so that leaky things like heap_getnext() are + * not called in a potentially long-lived context. + */ + oldcxt = MemoryContextSwitchTo(resultcxt); + dblist = lappend_oid(dblist, HeapTupleGetOid(tup)); + MemoryContextSwitchTo(oldcxt); + } + + heap_endscan(scan); + heap_close(rel, AccessShareLock); + + CommitTransactionCommand(); + + /* + * Check if database has foreign transaction entry. Delete entry from the + * list if the database has. + */ + for (cell = list_head(dblist); cell != NULL; cell = next) + { + Oid dbid = lfirst_oid(cell); + bool exists; + + next = lnext(cell); + + exists = fdw_xact_exists(InvalidTransactionId, dbid, InvalidOid, InvalidOid); + + if (!exists) + dblist = list_delete_cell(dblist, cell, prev); + else + prev = cell; + } + + return dblist; +} diff --git a/contrib/postgres_fdw/Makefile b/contrib/postgres_fdw/Makefile index 3543312..8054330 100644 --- a/contrib/postgres_fdw/Makefile +++ b/contrib/postgres_fdw/Makefile @@ -11,6 +11,7 @@ EXTENSION = postgres_fdw DATA = postgres_fdw--1.0.sql REGRESS = postgres_fdw +REGRESS_OPTS= --temp-config $(top_srcdir)/contrib/postgres_fdw/pgfdw.conf ifdef USE_PGXS PG_CONFIG = pg_config diff --git a/contrib/postgres_fdw/connection.c b/contrib/postgres_fdw/connection.c index be4ec07..0526094 100644 --- a/contrib/postgres_fdw/connection.c +++ b/contrib/postgres_fdw/connection.c @@ -16,7 +16,9 @@ #include "access/htup_details.h" #include "catalog/pg_user_mapping.h" +#include "access/fdw_xact.h" #include "access/xact.h" +#include "commands/defrem.h" #include "mb/pg_wchar.h" #include "miscadmin.h" #include "pgstat.h" @@ -73,12 +75,14 @@ static unsigned int prep_stmt_number = 0; static bool xact_got_connection = false; /* prototypes of private functions */ -static PGconn *connect_pg_server(ForeignServer *server, UserMapping *user); + +static PGconn *connect_pg_server(ForeignServer *server, UserMapping *user, + bool connection_error_ok); static void disconnect_pg_server(ConnCacheEntry *entry); static void check_conn_params(const char **keywords, const char **values); static void configure_remote_session(PGconn *conn); static void do_sql_command(PGconn *conn, const char *sql); -static void begin_remote_xact(ConnCacheEntry *entry); +static void begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid); static void pgfdw_xact_callback(XactEvent event, void *arg); static void pgfdw_subxact_callback(SubXactEvent event, SubTransactionId mySubid, @@ -91,6 +95,8 @@ static bool pgfdw_exec_cleanup_query(PGconn *conn, const char *query, bool ignore_errors); static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime, PGresult **result); +static bool server_uses_two_phase_commit(ForeignServer *server); +static void pgfdw_cleanup_after_transaction(ConnCacheEntry *entry); /* @@ -102,9 +108,20 @@ static bool pgfdw_get_cleanup_result(PGconn *conn, TimestampTz endtime, * will_prep_stmt must be true if caller intends to create any prepared * statements. Since those don't go away automatically at transaction end * (not even on error), we need this flag to cue manual cleanup. + * + * connection_error_ok if true, indicates that caller can handle connection + * error by itself. If false, raise error. + * + * XXX Note that caching connections theoretically requires a mechanism to + * detect change of FDW objects to invalidate already established connections. + * We could manage that by watching for invalidation events on the relevant + * syscaches. For the moment, though, it's not clear that this would really + * be useful and not mere pedantry. We could not flush any active connections + * mid-transaction anyway. */ PGconn * -GetConnection(UserMapping *user, bool will_prep_stmt) +GetConnection(UserMapping *user, bool will_prep_stmt, + bool start_transaction, bool connection_error_ok) { bool found; ConnCacheEntry *entry; @@ -136,9 +153,6 @@ GetConnection(UserMapping *user, bool will_prep_stmt) pgfdw_inval_callback, (Datum) 0); } - /* Set flag that we did GetConnection during the current transaction */ - xact_got_connection = true; - /* Create hash key for the entry. Assume no pad bytes in key struct */ key = user->umid; @@ -197,8 +211,20 @@ GetConnection(UserMapping *user, bool will_prep_stmt) GetSysCacheHashValue1(USERMAPPINGOID, ObjectIdGetDatum(user->umid)); - /* Now try to make the connection */ - entry->conn = connect_pg_server(server, user); + entry->conn = connect_pg_server(server, user, connection_error_ok); + + /* + * If the attempt to connect to the foreign server failed, we should not + * come here, unless the caller has indicated so. + */ + Assert(entry->conn || connection_error_ok); + + if (!entry->conn && connection_error_ok) + { + elog(DEBUG3, "attempt to connection to server \"%s\" by postgres_fdw failed", + server->servername); + return NULL; + } elog(DEBUG3, "new postgres_fdw connection %p for server \"%s\" (user mapping oid %u, userid %u)", entry->conn, server->servername, user->umid, user->userid); @@ -207,7 +233,12 @@ GetConnection(UserMapping *user, bool will_prep_stmt) /* * Start a new transaction or subtransaction if needed. */ - begin_remote_xact(entry); + if (start_transaction) + { + begin_remote_xact(entry, user->serverid, user->userid); + /* Set flag that we did GetConnection during the current transaction */ + xact_got_connection = true; + } /* Remember if caller will prepare statements */ entry->have_prep_stmt |= will_prep_stmt; @@ -217,9 +248,12 @@ GetConnection(UserMapping *user, bool will_prep_stmt) /* * Connect to remote server using specified server and user mapping properties. + * If the attempt to connect fails, and the caller can handle connection failure + * (connection_error_ok = true) return NULL, throw error otherwise. */ static PGconn * -connect_pg_server(ForeignServer *server, UserMapping *user) +connect_pg_server(ForeignServer *server, UserMapping *user, + bool connection_error_ok) { PGconn *volatile conn = NULL; @@ -265,11 +299,29 @@ connect_pg_server(ForeignServer *server, UserMapping *user) conn = PQconnectdbParams(keywords, values, false); if (!conn || PQstatus(conn) != CONNECTION_OK) - ereport(ERROR, - (errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), - errmsg("could not connect to server \"%s\"", - server->servername), - errdetail_internal("%s", pchomp(PQerrorMessage(conn))))); + { + char *connmessage; + int msglen; + + /* libpq typically appends a newline, strip that */ + connmessage = pstrdup(PQerrorMessage(conn)); + msglen = strlen(connmessage); + if (msglen > 0 && connmessage[msglen - 1] == '\n') + connmessage[msglen - 1] = '\0'; + + if (connection_error_ok) + { + return NULL; + } + else + { + ereport(ERROR, + (errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), + errmsg("could not connect to server \"%s\"", + server->servername), + errdetail_internal("%s", pchomp(PQerrorMessage(conn))))); + } + } /* * Check that non-superuser has used password to establish connection; @@ -414,15 +466,22 @@ do_sql_command(PGconn *conn, const char *sql) * control which remote queries share a snapshot. */ static void -begin_remote_xact(ConnCacheEntry *entry) +begin_remote_xact(ConnCacheEntry *entry, Oid serverid, Oid userid) { int curlevel = GetCurrentTransactionNestLevel(); + ForeignServer *server = GetForeignServer(serverid); /* Start main transaction if we haven't yet */ if (entry->xact_depth <= 0) { const char *sql; + /* + * Register the new foreign server and check whether the two phase + * compliance is possible. + */ + RegisterXactForeignServer(serverid, userid, server_uses_two_phase_commit(server)); + elog(DEBUG3, "starting remote transaction on connection %p", entry->conn); @@ -644,185 +703,284 @@ pgfdw_report_error(int elevel, PGresult *res, PGconn *conn, } /* + * postgresGetPrepareId + * + * The function crafts prepared transaction identifier. PostgreSQL documentation + * mentions two restrictions on the name + * 1. String literal, less than 200 bytes long. + * 2. Should not be same as any other concurrent prepared transaction id. + * + * To make the prepared transaction id, we should ideally use something like + * UUID, which gives unique ids with high probability, but that may be expensive + * here and UUID extension which provides the function to generate UUID is + * not part of the core. + */ +extern char * +postgresGetPrepareId(Oid serverid, Oid userid, int *prep_info_len) +{ +/* Maximum length of the prepared transaction id, borrowed from twophase.c */ +#define PREP_XACT_ID_MAX_LEN 200 +#define RANDOM_LARGE_MULTIPLIER 1000 + char *prep_info; + + /* Allocate the memory in the same context as the hash entry */ + prep_info = (char *)palloc(PREP_XACT_ID_MAX_LEN * sizeof(char)); + snprintf(prep_info, PREP_XACT_ID_MAX_LEN, "%s_%4ld_%d_%d", + "px", Abs(random() * RANDOM_LARGE_MULTIPLIER), + serverid, userid); + /* Account for the last NULL byte */ + *prep_info_len = strlen(prep_info); + return prep_info; +} + +/* + * postgresPrepareForeignTransaction + * + * The function prepares transaction on foreign server. + */ +bool +postgresPrepareForeignTransaction(Oid serverid, Oid userid, Oid umid, + int prep_info_len, char *prep_info) +{ + StringInfo command; + PGresult *res; + ConnCacheEntry *entry = NULL; + ConnCacheKey key; + bool found; + + /* Create hash key for the entry. Assume no pad bytes in key struct */ + key = umid; + + Assert(ConnectionHash); + entry = hash_search(ConnectionHash, &key, HASH_FIND, &found); + + if (found && entry->conn) + { + bool result; + PGconn *conn = entry->conn; + + command = makeStringInfo(); + appendStringInfo(command, "PREPARE TRANSACTION '%.*s'", prep_info_len, + prep_info); + res = PQexec(conn, command->data); + result = (PQresultStatus(res) == PGRES_COMMAND_OK); + + if (!result) + { + /* + * TODO: check whether we should raise an error or warning. + * The command failed, raise a warning, so that the reason for + * failure gets logged. Do not raise an error, the caller i.e. foreign + * transaction manager takes care of taking appropriate action. + */ + pgfdw_report_error(WARNING, res, conn, false, command->data); + } + + PQclear(res); + pgfdw_cleanup_after_transaction(entry); + return result; + } + else + return false; +} + +bool +postgresEndForeignTransaction(Oid serverid, Oid userid, Oid umid, bool is_commit) +{ + StringInfo command; + PGresult *res; + ConnCacheEntry *entry = NULL; + ConnCacheKey key; + bool found; + + /* Create hash key for the entry. Assume no pad bytes in key struct */ + key = umid; + + Assert(ConnectionHash); + entry = hash_search(ConnectionHash, &key, HASH_FIND, &found); + + if (found && entry->conn) + { + PGconn *conn = entry->conn; + bool result; + + command = makeStringInfo(); + appendStringInfo(command, "%s TRANSACTION", + is_commit ? "COMMIT" : "ROLLBACK"); + res = PQexec(conn, command->data); + result = (PQresultStatus(res) == PGRES_COMMAND_OK); + if (!result) + { + /* + * The local transaction has ended, so there is no point in raising + * error. Raise a warning so that the reason for the failure gets + * logged. + */ + pgfdw_report_error(WARNING, res, conn, false, command->data); + } + + PQclear(res); + pgfdw_cleanup_after_transaction(entry); + return result; + } + return false; +} + +/* + * postgresResolvePreparedForeignTransaction + * + * The function commit or abort prepared transaction on foreign server. + * This function could be called when we don't have any connections to the + * foreign server involving distributed transaction being resolved. + */ +bool +postgresResolvePreparedForeignTransaction(Oid serverid, Oid userid, Oid umid, + bool is_commit, + int prep_info_len, char *prep_info) +{ + PGconn *conn = NULL; + + /* + * If there exists a connection in the connection cache that can be used, + * use it. If there is none, we need foreign server and user information + * which can be obtained only when in a transaction block. + * If we are resolving prepared foreign transactions immediately after + * preparing them, the connection hash would have a connection. If we are + * resolving them any other time, a resolver would have started a + * transaction. + */ + if (ConnectionHash) + { + /* Connection hash should have a connection we want */ + bool found; + ConnCacheKey key; + ConnCacheEntry *entry; + + /* Create hash key for the entry. Assume no pad bytes in key struct */ + key = umid; + + entry = (ConnCacheEntry *)hash_search(ConnectionHash, &key, HASH_FIND, &found); + if (found && entry->conn) + conn = entry->conn; + } + + if (!conn && IsTransactionState()) + conn = GetConnection(GetUserMapping(userid, serverid), false, false, true); + + /* Proceed with resolution if we got a connection, else return false */ + if (conn) + { + StringInfo command; + PGresult *res; + bool result; + + command = makeStringInfo(); + appendStringInfo(command, "%s PREPARED '%.*s'", + is_commit ? "COMMIT" : "ROLLBACK", + prep_info_len, prep_info); + res = PQexec(conn, command->data); + + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + int sqlstate; + char *diag_sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE); + /* + * The command failed, raise a warning to log the reason of failure. + * We may not be in a transaction here, so raising error doesn't + * help. Even if we are in a transaction, it would be the resolver + * transaction, which will get aborted on raising error, thus + * delaying resolution of other prepared foreign transactions. + */ + pgfdw_report_error(WARNING, res, conn, false, command->data); + + if (diag_sqlstate) + { + sqlstate = MAKE_SQLSTATE(diag_sqlstate[0], + diag_sqlstate[1], + diag_sqlstate[2], + diag_sqlstate[3], + diag_sqlstate[4]); + } + else + sqlstate = ERRCODE_CONNECTION_FAILURE; + + /* + * If we tried to COMMIT/ABORT a prepared transaction and the prepared + * transaction was missing on the foreign server, it was probably + * resolved by some other means. Anyway, it should be considered as resolved. + */ + result = (sqlstate == ERRCODE_UNDEFINED_OBJECT); + } + else + result = true; + + PQclear(res); + ReleaseConnection(conn); + return result; + } + else + return false; +} + +static void +pgfdw_cleanup_after_transaction(ConnCacheEntry *entry) +{ + /* + * If there were any errors in subtransactions, and we made prepared + * statements, do a DEALLOCATE ALL to make sure we get rid of all + * prepared statements. This is annoying and not terribly bulletproof, + * but it's probably not worth trying harder. + * + * DEALLOCATE ALL only exists in 8.3 and later, so this constrains how + * old a server postgres_fdw can communicate with. We intentionally + * ignore errors in the DEALLOCATE, so that we can hobble along to some + * extent with older servers (leaking prepared statements as we go; + * but we don't really support update operations pre-8.3 anyway). + */ + if (entry->have_prep_stmt && entry->have_error) + { + PGresult *res = PQexec(entry->conn, "DEALLOCATE ALL"); + PQclear(res); + } + + entry->have_prep_stmt = false; + entry->have_error = false; + /* Reset state to show we're out of a transaction */ + entry->xact_depth = 0; + + /* + * If the connection isn't in a good idle state, discard it to + * recover. Next GetConnection will open a new connection. + */ + if (PQstatus(entry->conn) != CONNECTION_OK || + PQtransactionStatus(entry->conn) != PQTRANS_IDLE) + { + elog(DEBUG3, "discarding connection %p", entry->conn); + PQfinish(entry->conn); + entry->conn = NULL; + } + + /* + * TODO: these next two statements should be moved to end of transaction + * call back. + * Regardless of the event type, we can now mark ourselves as out of the + * transaction. + */ + xact_got_connection = false; + + /* Also reset cursor numbering for next transaction */ + cursor_number = 0; +} + +/* * pgfdw_xact_callback --- cleanup at main-transaction end. */ static void pgfdw_xact_callback(XactEvent event, void *arg) { - HASH_SEQ_STATUS scan; - ConnCacheEntry *entry; - - /* Quick exit if no connections were touched in this transaction. */ - if (!xact_got_connection) - return; - - /* - * Scan all connection cache entries to find open remote transactions, and - * close them. - */ - hash_seq_init(&scan, ConnectionHash); - while ((entry = (ConnCacheEntry *) hash_seq_search(&scan))) - { - PGresult *res; - - /* Ignore cache entry if no open connection right now */ - if (entry->conn == NULL) - continue; - - /* If it has an open remote transaction, try to close it */ - if (entry->xact_depth > 0) - { - bool abort_cleanup_failure = false; - - elog(DEBUG3, "closing remote transaction on connection %p", - entry->conn); - - switch (event) - { - case XACT_EVENT_PARALLEL_PRE_COMMIT: - case XACT_EVENT_PRE_COMMIT: - - /* - * If abort cleanup previously failed for this connection, - * we can't issue any more commands against it. - */ - pgfdw_reject_incomplete_xact_state_change(entry); - - /* Commit all remote transactions during pre-commit */ - entry->changing_xact_state = true; - do_sql_command(entry->conn, "COMMIT TRANSACTION"); - entry->changing_xact_state = false; - - /* - * If there were any errors in subtransactions, and we - * made prepared statements, do a DEALLOCATE ALL to make - * sure we get rid of all prepared statements. This is - * annoying and not terribly bulletproof, but it's - * probably not worth trying harder. - * - * DEALLOCATE ALL only exists in 8.3 and later, so this - * constrains how old a server postgres_fdw can - * communicate with. We intentionally ignore errors in - * the DEALLOCATE, so that we can hobble along to some - * extent with older servers (leaking prepared statements - * as we go; but we don't really support update operations - * pre-8.3 anyway). - */ - if (entry->have_prep_stmt && entry->have_error) - { - res = PQexec(entry->conn, "DEALLOCATE ALL"); - PQclear(res); - } - entry->have_prep_stmt = false; - entry->have_error = false; - break; - case XACT_EVENT_PRE_PREPARE: - - /* - * We disallow remote transactions that modified anything, - * since it's not very reasonable to hold them open until - * the prepared transaction is committed. For the moment, - * throw error unconditionally; later we might allow - * read-only cases. Note that the error will cause us to - * come right back here with event == XACT_EVENT_ABORT, so - * we'll clean up the connection state at that point. - */ - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot prepare a transaction that modified remote tables"))); - break; - case XACT_EVENT_PARALLEL_COMMIT: - case XACT_EVENT_COMMIT: - case XACT_EVENT_PREPARE: - /* Pre-commit should have closed the open transaction */ - elog(ERROR, "missed cleaning up connection during pre-commit"); - break; - case XACT_EVENT_PARALLEL_ABORT: - case XACT_EVENT_ABORT: - - /* - * Don't try to clean up the connection if we're already - * in error recursion trouble. - */ - if (in_error_recursion_trouble()) - entry->changing_xact_state = true; - - /* - * If connection is already unsalvageable, don't touch it - * further. - */ - if (entry->changing_xact_state) - break; - - /* - * Mark this connection as in the process of changing - * transaction state. - */ - entry->changing_xact_state = true; - - /* Assume we might have lost track of prepared statements */ - entry->have_error = true; - - /* - * If a command has been submitted to the remote server by - * using an asynchronous execution function, the command - * might not have yet completed. Check to see if a - * command is still being processed by the remote server, - * and if so, request cancellation of the command. - */ - if (PQtransactionStatus(entry->conn) == PQTRANS_ACTIVE && - !pgfdw_cancel_query(entry->conn)) - { - /* Unable to cancel running query. */ - abort_cleanup_failure = true; - } - else if (!pgfdw_exec_cleanup_query(entry->conn, - "ABORT TRANSACTION", - false)) - { - /* Unable to abort remote transaction. */ - abort_cleanup_failure = true; - } - else if (entry->have_prep_stmt && entry->have_error && - !pgfdw_exec_cleanup_query(entry->conn, - "DEALLOCATE ALL", - true)) - { - /* Trouble clearing prepared statements. */ - abort_cleanup_failure = true; - } - else - { - entry->have_prep_stmt = false; - entry->have_error = false; - } - - /* Disarm changing_xact_state if it all worked. */ - entry->changing_xact_state = abort_cleanup_failure; - break; - } - } - - /* Reset state to show we're out of a transaction */ - entry->xact_depth = 0; - - /* - * If the connection isn't in a good idle state, discard it to - * recover. Next GetConnection will open a new connection. - */ - if (PQstatus(entry->conn) != CONNECTION_OK || - PQtransactionStatus(entry->conn) != PQTRANS_IDLE || - entry->changing_xact_state) - { - elog(DEBUG3, "discarding connection %p", entry->conn); - disconnect_pg_server(entry); - } - } - /* * Regardless of the event type, we can now mark ourselves as out of the - * transaction. (Note: if we are here during PRE_COMMIT or PRE_PREPARE, - * this saves a useless scan of the hashtable during COMMIT or PREPARE.) + * transction. */ xact_got_connection = false; @@ -1193,3 +1351,25 @@ exit: ; *result = last_res; return timed_out; } +/* + * server_uses_two_phase_commit + * Returns true if the foreign server is configured to support 2PC. + */ +static bool +server_uses_two_phase_commit(ForeignServer *server) +{ + ListCell *lc; + + /* Check the options for two phase compliance */ + foreach(lc, server->options) + { + DefElem *d = (DefElem *) lfirst(lc); + + if (strcmp(d->defname, "two_phase_commit") == 0) + { + return defGetBoolean(d); + } + } + /* By default a server is not 2PC compliant */ + return false; +} diff --git a/contrib/postgres_fdw/expected/postgres_fdw.out b/contrib/postgres_fdw/expected/postgres_fdw.out index c19b331..b5d0f13 100644 --- a/contrib/postgres_fdw/expected/postgres_fdw.out +++ b/contrib/postgres_fdw/expected/postgres_fdw.out @@ -13,12 +13,17 @@ DO $d$ OPTIONS (dbname '$$||current_database()||$$', port '$$||current_setting('port')||$$' )$$; + EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw + OPTIONS (dbname '$$||current_database()||$$', + port '$$||current_setting('port')||$$' + )$$; END; $d$; CREATE USER MAPPING FOR public SERVER testserver1 OPTIONS (user 'value', password 'value'); CREATE USER MAPPING FOR CURRENT_USER SERVER loopback; CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2; +CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3; -- =================================================================== -- create objects used through FDW loopback server -- =================================================================== @@ -52,6 +57,14 @@ CREATE TABLE "S 1"."T 4" ( c3 text, CONSTRAINT t4_pkey PRIMARY KEY (c1) ); +CREATE TABLE "S 1"."T 5" ( + c1 int NOT NULL, + CONSTRAINT t5_pkey PRIMARY KEY (c1) +); +CREATE TABLE "S 1"."T 6" ( + c1 int NOT NULL, + CONSTRAINT t6_pkey PRIMARY KEY (c1) +); INSERT INTO "S 1"."T 1" SELECT id, id % 10, @@ -78,10 +91,13 @@ INSERT INTO "S 1"."T 4" 'AAA' || to_char(id, 'FM000') FROM generate_series(1, 100) id; DELETE FROM "S 1"."T 4" WHERE c1 % 3 != 0; -- delete for outer join tests +INSERT INTO "S 1"."T 5" + SELECT generate_series(1, 100); ANALYZE "S 1"."T 1"; ANALYZE "S 1"."T 2"; ANALYZE "S 1"."T 3"; ANALYZE "S 1"."T 4"; +ANALYZE "S 1"."T 5"; -- =================================================================== -- create foreign tables -- =================================================================== @@ -124,6 +140,15 @@ CREATE FOREIGN TABLE ft6 ( c2 int NOT NULL, c3 text ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4'); +CREATE FOREIGN TABLE ft7 ( + c1 int NOT NULL +) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5'); +CREATE FOREIGN TABLE ft8 ( + c1 int NOT NULL +) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5'); +CREATE FOREIGN TABLE ft9 ( + c1 int NOT NULL +) SERVER loopback3 OPTIONS (schema_name 'S 1', table_name 'T 5'); -- A table with oids. CREATE FOREIGN TABLE doesn't support the -- WITH OIDS option, but ALTER does. CREATE FOREIGN TABLE ft_pg_type ( @@ -188,8 +213,11 @@ ALTER FOREIGN TABLE ft2 ALTER COLUMN c1 OPTIONS (column_name 'C 1'); public | ft4 | loopback | (schema_name 'S 1', table_name 'T 3') | public | ft5 | loopback | (schema_name 'S 1', table_name 'T 4') | public | ft6 | loopback2 | (schema_name 'S 1', table_name 'T 4') | + public | ft7 | loopback | (schema_name 'S 1', table_name 'T 5') | + public | ft8 | loopback2 | (schema_name 'S 1', table_name 'T 5') | + public | ft9 | loopback3 | (schema_name 'S 1', table_name 'T 5') | public | ft_pg_type | loopback | (schema_name 'pg_catalog', table_name 'pg_type') | -(6 rows) +(9 rows) -- Test that alteration of server options causes reconnection -- Remote's errors might be non-English, so hide them to ensure stable results @@ -7346,3 +7374,129 @@ AND ftoptions @> array['fetch_size=60000']; (1 row) ROLLBACK; +-- =================================================================== +-- test Atomic commit across foreign servers +-- =================================================================== +ALTER SERVER loopback OPTIONS(ADD two_phase_commit 'off'); +ALTER SERVER loopback2 OPTIONS(ADD two_phase_commit 'on'); +ALTER SERVER loopback3 OPTIONS(ADD two_phase_commit 'on'); +-- one server not supporting 2PC. +BEGIN; +INSERT INTO ft7 VALUES(101); +COMMIT; +SELECT COUNT(*) FROM ft8; + count +------- + 101 +(1 row) + +-- One server supporting 2PC and another one server not supporting 2PC. +BEGIN; +INSERT INTO ft7 VALUES(102); +INSERT INTO ft8 VALUES(103); +COMMIT; +SELECT COUNT(*) FROM ft8; + count +------- + 103 +(1 row) + +-- Two supporting server. +BEGIN; +INSERT INTO ft8 VALUES(105); +INSERT INTO ft9 VALUES(106); +COMMIT; +SELECT COUNT(*) FROM ft8; + count +------- + 105 +(1 row) + +-- Local changes and two servers supporting 2PC. +BEGIN; +INSERT INTO ft7 VALUES(110); +INSERT INTO ft8 VALUES(111); +INSERT INTO ft9 VALUES(112); +INSERT INTO "S 1"."T 6" VALUES (3); +COMMIT; +SELECT COUNT(*) FROM ft8; + count +------- + 108 +(1 row) + +SELECT COUNT(*) FROM "S 1"."T 6"; + count +------- + 1 +(1 row) + +-- transaction updating on single supporting foreign server with violation on foreign server. +BEGIN; +INSERT INTO ft8 VALUES(113); +INSERT INTO ft8 VALUES(110); -- violation on foreign server +ERROR: duplicate key value violates unique constraint "t5_pkey" +DETAIL: Key (c1)=(110) already exists. +CONTEXT: Remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1) +COMMIT; +SELECT COUNT(*) FROM ft8; + count +------- + 108 +(1 row) + +-- transaction updating on single supporting foreign server and local with violation on local. +BEGIN; +INSERT INTO ft8 VALUES(114); +INSERT INTO "S 1"."T 6" VALUES (4); +INSERT INTO "S 1"."T 6" VALUES (3); -- violation on local +ERROR: duplicate key value violates unique constraint "t6_pkey" +DETAIL: Key (c1)=(3) already exists. +COMMIT; +SELECT COUNT(*) FROM ft8; + count +------- + 108 +(1 row) + +SELECT COUNT(*) FROM "S 1"."T 6"; + count +------- + 1 +(1 row) + +-- violation on foreign server supporting 2PC. +BEGIN; +INSERT INTO ft8 VALUES(115); +INSERT INTO ft9 VALUES(116); +INSERT INTO ft9 VALUES(110); -- violation on foreign server +ERROR: duplicate key value violates unique constraint "t5_pkey" +DETAIL: Key (c1)=(110) already exists. +CONTEXT: Remote SQL command: INSERT INTO "S 1"."T 5"(c1) VALUES ($1) +COMMIT; +SELECT COUNT(*) FROM ft8; + count +------- + 108 +(1 row) + +-- transaction involing local and foreign server with violation on local server. +BEGIN; +INSERT INTO ft8 VALUES(117); +INSERT INTO ft9 VALUES(118); +INSERT INTO "S 1"."T 6" VALUES (3); -- violation on local +ERROR: duplicate key value violates unique constraint "t6_pkey" +DETAIL: Key (c1)=(3) already exists. +COMMIT; +SELECT COUNT(*) FROM ft8; + count +------- + 108 +(1 row) + +SELECT COUNT(*) FROM "S 1"."T 6"; + count +------- + 1 +(1 row) + diff --git a/contrib/postgres_fdw/option.c b/contrib/postgres_fdw/option.c index 67e1c59..67e1127 100644 --- a/contrib/postgres_fdw/option.c +++ b/contrib/postgres_fdw/option.c @@ -108,7 +108,8 @@ postgres_fdw_validator(PG_FUNCTION_ARGS) * Validate option value, when we can do so without any context. */ if (strcmp(def->defname, "use_remote_estimate") == 0 || - strcmp(def->defname, "updatable") == 0) + strcmp(def->defname, "updatable") == 0 || + strcmp(def->defname, "two_phase_commit") == 0) { /* these accept only boolean values */ (void) defGetBoolean(def); @@ -177,6 +178,8 @@ InitPgFdwOptions(void) /* fetch_size is available on both server and table */ {"fetch_size", ForeignServerRelationId, false}, {"fetch_size", ForeignTableRelationId, false}, + /* two phase commit support */ + {"two_phase_commit", ForeignServerRelationId, false}, {NULL, InvalidOid, false} }; diff --git a/contrib/postgres_fdw/pg_fdw.conf b/contrib/postgres_fdw/pg_fdw.conf new file mode 100644 index 0000000..b086227 --- /dev/null +++ b/contrib/postgres_fdw/pg_fdw.conf @@ -0,0 +1,2 @@ +ax_prepared_foreign_transactions = 100 +max_prepared_transactions = 10 diff --git a/contrib/postgres_fdw/pgfdw.conf b/contrib/postgres_fdw/pgfdw.conf new file mode 100644 index 0000000..2184040 --- /dev/null +++ b/contrib/postgres_fdw/pgfdw.conf @@ -0,0 +1,2 @@ +max_prepared_transactions = 10 +max_prepared_foreign_transactions = 10 diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index fb65e2e..988f4c6 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -14,6 +14,8 @@ #include "postgres_fdw.h" +#include "access/fdw_xact.h" +#include "access/xact.h" #include "access/htup_details.h" #include "access/sysattr.h" #include "catalog/pg_class.h" @@ -469,6 +471,12 @@ postgres_fdw_handler(PG_FUNCTION_ARGS) /* Support functions for join push-down */ routine->GetForeignJoinPaths = postgresGetForeignJoinPaths; + /* Support functions for foreign transactions */ + routine->GetPrepareId = postgresGetPrepareId; + routine->PrepareForeignTransaction = postgresPrepareForeignTransaction; + routine->ResolvePreparedForeignTransaction = postgresResolvePreparedForeignTransaction; + routine->EndForeignTransaction = postgresEndForeignTransaction; + /* Support functions for upper relation push-down */ routine->GetForeignUpperPaths = postgresGetForeignUpperPaths; @@ -1322,7 +1330,7 @@ postgresBeginForeignScan(ForeignScanState *node, int eflags) * Get connection to the foreign server. Connection manager will * establish new connection if necessary. */ - fsstate->conn = GetConnection(user, false); + fsstate->conn = GetConnection(user, false, true, false); /* Assign a unique ID for my cursor */ fsstate->cursor_number = GetCursorNumber(fsstate->conn); @@ -1700,7 +1708,7 @@ postgresBeginForeignModify(ModifyTableState *mtstate, user = GetUserMapping(userid, table->serverid); /* Open connection; report that we'll create a prepared statement. */ - fmstate->conn = GetConnection(user, true); + fmstate->conn = GetConnection(user, true, true, false); fmstate->p_name = NULL; /* prepared statement not made yet */ /* Deconstruct fdw_private data. */ @@ -2303,7 +2311,7 @@ postgresBeginDirectModify(ForeignScanState *node, int eflags) * Get connection to the foreign server. Connection manager will * establish new connection if necessary. */ - dmstate->conn = GetConnection(user, false); + dmstate->conn = GetConnection(user, false, true, false); /* Initialize state variable */ dmstate->num_tuples = -1; /* -1 means not set yet */ @@ -2564,7 +2572,7 @@ estimate_path_cost_size(PlannerInfo *root, &retrieved_attrs, NULL); /* Get the remote estimate */ - conn = GetConnection(fpinfo->user, false); + conn = GetConnection(fpinfo->user, false, true, false); get_remote_estimate(sql.data, conn, &rows, &width, &startup_cost, &total_cost); ReleaseConnection(conn); @@ -3501,7 +3509,7 @@ postgresAnalyzeForeignTable(Relation relation, */ table = GetForeignTable(RelationGetRelid(relation)); user = GetUserMapping(relation->rd_rel->relowner, table->serverid); - conn = GetConnection(user, false); + conn = GetConnection(user, false, true, false); /* * Construct command to get page count for relation. @@ -3591,7 +3599,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel, table = GetForeignTable(RelationGetRelid(relation)); server = GetForeignServer(table->serverid); user = GetUserMapping(relation->rd_rel->relowner, table->serverid); - conn = GetConnection(user, false); + conn = GetConnection(user, false, true, false); /* * Construct cursor that retrieves whole rows from remote. @@ -3814,7 +3822,7 @@ postgresImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid) */ server = GetForeignServer(serverOid); mapping = GetUserMapping(GetUserId(), server->serverid); - conn = GetConnection(mapping, false); + conn = GetConnection(mapping, false, true, false); /* Don't attempt to import collation if remote server hasn't got it */ if (PQserverVersion(conn) < 90100) diff --git a/contrib/postgres_fdw/postgres_fdw.h b/contrib/postgres_fdw/postgres_fdw.h index 788b003..0213fdb 100644 --- a/contrib/postgres_fdw/postgres_fdw.h +++ b/contrib/postgres_fdw/postgres_fdw.h @@ -13,6 +13,7 @@ #ifndef POSTGRES_FDW_H #define POSTGRES_FDW_H +#include "access/fdw_xact.h" #include "foreign/foreign.h" #include "lib/stringinfo.h" #include "nodes/relation.h" @@ -115,7 +116,8 @@ extern int set_transmission_modes(void); extern void reset_transmission_modes(int nestlevel); /* in connection.c */ -extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt); +extern PGconn *GetConnection(UserMapping *user, bool will_prep_stmt, + bool start_transaction, bool connection_error_ok); extern void ReleaseConnection(PGconn *conn); extern unsigned int GetCursorNumber(PGconn *conn); extern unsigned int GetPrepStmtNumber(PGconn *conn); @@ -177,6 +179,15 @@ extern void deparseSelectStmtForRel(StringInfo buf, PlannerInfo *root, List *remote_conds, List *pathkeys, bool is_subquery, List **retrieved_attrs, List **params_list); extern const char *get_jointype_name(JoinType jointype); +extern char *postgresGetPrepareId(Oid serveroid, Oid userid, int *prep_info_len); +extern bool postgresPrepareForeignTransaction(Oid serverid, Oid userid, + Oid umid, int prep_info_len, + char *prep_info); +extern bool postgresResolvePreparedForeignTransaction(Oid serverid, Oid userid, + Oid umid, bool is_commit, + int prep_info_len, char *prep_info); +extern bool postgresEndForeignTransaction(Oid serverid, Oid userid, Oid umid, bool is_commit); + /* in shippable.c */ extern bool is_builtin(Oid objectId); diff --git a/contrib/postgres_fdw/sql/postgres_fdw.sql b/contrib/postgres_fdw/sql/postgres_fdw.sql index 5f65d9d..68daa86 100644 --- a/contrib/postgres_fdw/sql/postgres_fdw.sql +++ b/contrib/postgres_fdw/sql/postgres_fdw.sql @@ -15,6 +15,10 @@ DO $d$ OPTIONS (dbname '$$||current_database()||$$', port '$$||current_setting('port')||$$' )$$; + EXECUTE $$CREATE SERVER loopback3 FOREIGN DATA WRAPPER postgres_fdw + OPTIONS (dbname '$$||current_database()||$$', + port '$$||current_setting('port')||$$' + )$$; END; $d$; @@ -22,6 +26,7 @@ CREATE USER MAPPING FOR public SERVER testserver1 OPTIONS (user 'value', password 'value'); CREATE USER MAPPING FOR CURRENT_USER SERVER loopback; CREATE USER MAPPING FOR CURRENT_USER SERVER loopback2; +CREATE USER MAPPING FOR CURRENT_USER SERVER loopback3; -- =================================================================== -- create objects used through FDW loopback server @@ -56,6 +61,15 @@ CREATE TABLE "S 1"."T 4" ( c3 text, CONSTRAINT t4_pkey PRIMARY KEY (c1) ); +CREATE TABLE "S 1"."T 5" ( + c1 int NOT NULL, + CONSTRAINT t5_pkey PRIMARY KEY (c1) +); + +CREATE TABLE "S 1"."T 6" ( + c1 int NOT NULL, + CONSTRAINT t6_pkey PRIMARY KEY (c1) +); INSERT INTO "S 1"."T 1" SELECT id, @@ -83,11 +97,14 @@ INSERT INTO "S 1"."T 4" 'AAA' || to_char(id, 'FM000') FROM generate_series(1, 100) id; DELETE FROM "S 1"."T 4" WHERE c1 % 3 != 0; -- delete for outer join tests +INSERT INTO "S 1"."T 5" + SELECT generate_series(1, 100); ANALYZE "S 1"."T 1"; ANALYZE "S 1"."T 2"; ANALYZE "S 1"."T 3"; ANALYZE "S 1"."T 4"; +ANALYZE "S 1"."T 5"; -- =================================================================== -- create foreign tables @@ -136,6 +153,19 @@ CREATE FOREIGN TABLE ft6 ( c3 text ) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 4'); +CREATE FOREIGN TABLE ft7 ( + c1 int NOT NULL +) SERVER loopback OPTIONS (schema_name 'S 1', table_name 'T 5'); + +CREATE FOREIGN TABLE ft8 ( + c1 int NOT NULL +) SERVER loopback2 OPTIONS (schema_name 'S 1', table_name 'T 5'); + +CREATE FOREIGN TABLE ft9 ( + c1 int NOT NULL +) SERVER loopback3 OPTIONS (schema_name 'S 1', table_name 'T 5'); + + -- A table with oids. CREATE FOREIGN TABLE doesn't support the -- WITH OIDS option, but ALTER does. CREATE FOREIGN TABLE ft_pg_type ( @@ -1764,3 +1794,76 @@ WHERE ftrelid = 'table30000'::regclass AND ftoptions @> array['fetch_size=60000']; ROLLBACK; + + +-- =================================================================== +-- test Atomic commit across foreign servers +-- =================================================================== + +ALTER SERVER loopback OPTIONS(ADD two_phase_commit 'off'); +ALTER SERVER loopback2 OPTIONS(ADD two_phase_commit 'on'); +ALTER SERVER loopback3 OPTIONS(ADD two_phase_commit 'on'); + + +-- one server not supporting 2PC. +BEGIN; +INSERT INTO ft7 VALUES(101); +COMMIT; +SELECT COUNT(*) FROM ft8; + +-- One server supporting 2PC and another one server not supporting 2PC. +BEGIN; +INSERT INTO ft7 VALUES(102); +INSERT INTO ft8 VALUES(103); +COMMIT; +SELECT COUNT(*) FROM ft8; + +-- Two supporting server. +BEGIN; +INSERT INTO ft8 VALUES(105); +INSERT INTO ft9 VALUES(106); +COMMIT; +SELECT COUNT(*) FROM ft8; + +-- Local changes and two servers supporting 2PC. +BEGIN; +INSERT INTO ft7 VALUES(110); +INSERT INTO ft8 VALUES(111); +INSERT INTO ft9 VALUES(112); +INSERT INTO "S 1"."T 6" VALUES (3); +COMMIT; +SELECT COUNT(*) FROM ft8; +SELECT COUNT(*) FROM "S 1"."T 6"; + +-- transaction updating on single supporting foreign server with violation on foreign server. +BEGIN; +INSERT INTO ft8 VALUES(113); +INSERT INTO ft8 VALUES(110); -- violation on foreign server +COMMIT; +SELECT COUNT(*) FROM ft8; + +-- transaction updating on single supporting foreign server and local with violation on local. +BEGIN; +INSERT INTO ft8 VALUES(114); +INSERT INTO "S 1"."T 6" VALUES (4); +INSERT INTO "S 1"."T 6" VALUES (3); -- violation on local +COMMIT; +SELECT COUNT(*) FROM ft8; +SELECT COUNT(*) FROM "S 1"."T 6"; + +-- violation on foreign server supporting 2PC. +BEGIN; +INSERT INTO ft8 VALUES(115); +INSERT INTO ft9 VALUES(116); +INSERT INTO ft9 VALUES(110); -- violation on foreign server +COMMIT; +SELECT COUNT(*) FROM ft8; + +-- transaction involing local and foreign server with violation on local server. +BEGIN; +INSERT INTO ft8 VALUES(117); +INSERT INTO ft9 VALUES(118); +INSERT INTO "S 1"."T 6" VALUES (3); -- violation on local +COMMIT; +SELECT COUNT(*) FROM ft8; +SELECT COUNT(*) FROM "S 1"."T 6"; diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 4b265d9..7ddbf67 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -1451,6 +1451,27 @@ include_dir 'conf.d' + + max_prepared_foreign_transactions (integer) + + max_prepared_foreign_transactions configuration parameter + + + + + Sets the maximum number of foreign transactions that can be prepared + simultaneously. + This parameter can only be set at server start. + + + + When running a standby server, you must set this parameter to the + same or higher value than on the master server. Otherwise, queries + will not be allowed in the standby server. + + + + work_mem (integer) diff --git a/doc/src/sgml/contrib.sgml b/doc/src/sgml/contrib.sgml index f32b8a8..29a496b 100644 --- a/doc/src/sgml/contrib.sgml +++ b/doc/src/sgml/contrib.sgml @@ -115,6 +115,7 @@ CREATE EXTENSION module_name FROM unpackaged; &dict-int; &dict-xsyn; &earthdistance; + $fdw-transaction-resolver; &file-fdw; &fuzzystrmatch; &hstore; diff --git a/doc/src/sgml/fdwhandler.sgml b/doc/src/sgml/fdwhandler.sgml index a59e03a..f1e9a4f 100644 --- a/doc/src/sgml/fdwhandler.sgml +++ b/doc/src/sgml/fdwhandler.sgml @@ -1736,5 +1736,92 @@ GetForeignServerByName(const char *name, bool missing_ok); + + Transaction manager for Foreign Data Wrappers + + + PostgreSQL transaction manager allows FDWs to read and write + data on foreign server within a transaction while maintaining atomicity + (and hence consistency) of the foreign data. Every Foreign Data Wrapper is + required to register the foreign server along with the PostgreSQL + user whose user mapping is used to connect to the foreign server while starting a + transaction on the foreign server as part of the transaction on + PostgreSQL using RegisterXactForeignServer. + +void +RegisterXactForeignServer(Oid serverid, + Oid userid, + bool two_phase_compliant) + + two_phase_compliant should be true if the foreign server supports + two-phase commit protocol, false otherwise. + + + + An example of such transaction is as follows + +BEGIN; +UPDATE ft1 SET col = 'a'; +UPDATE ft2 SET col = 'b'; +COMMIT; + + ft1 and ft2 are foreign tables on different foreign servers may be using different + Foreign Data Wrappers. + + + + When max_prepared_foreign_transactions is more than zero + PostgreSQL employs Two-phase commit protocol to achieve + atomic distributed transaction. All the foreign servers registered should + support two-phase commit protocol. The two-phase commit protocol is used for + achieving atomic distributed transaction when more than two foreign servers + that support two-phase commit protocol are involved with transaction, or + transaction involves with one foreign server that support two-phase commit + protocol and changes on local data. In other case, for example where only one + foreign server that support two-phase commit is involved with transaction, + the two-phase commit protocol is not used. In Two-phase commit protocol is + processed in two phases: prepare phase and commit phase. In prepare phase, + PostgreSQL prepares the transactions on all the foreign + servers registered using RegisterXactForeignServer. If any of + the foreign server fails to prepare transaction, prepare phase fails. In commit + phase, all the prepared transactions are committed if prepare phase has succeeded + or rolled back if prepare phase fails to prepare transactions on all the foreign + servers. + + + + During prepare phase the distributed transaction manager calls + GetPrepareId to get the prepared transaction identifier for + each foreign server involved. It stores this identifier along with the + serverid and userid for later use. It then calls + ResolvePreparedForeignTranscation with the same identifier + with action FDW_XACT_RESOLVED. + + + + During commit phase the distributed transaction manager calls + ResolveForeignTransaction with the same identifier with action + FDW_XACT_COMMITTING_PREPARED to commit the prepared transaction or + FDW_XACT_ABORTING_PREPARED to rollback the prepared transaction. In case the + distributed transaction manager fails to commit or rollback a prepared + transaction because of connection failure, the operation can be tried again + through built-in pg_fdw_xact. One may set up a background worker + process to retry the operation by installing extension fdw_transaction_resolver + and including $libdir/fdw_transaction_resolver.so in + shared_preload_libraries. + + + + When max_prepared_foreign_transaction is zero, atomicity commit can + not be guaranteed across foreign servers. If transaction on PostgreSQL + is committed, Distributed transaction manager commit the transaction on all the + foreign servers registered using RegisterXactForeignServer, + independent of the outcome of the same operation on other foreign servers. + Thus transactions on some foreign servers may be committed, while the same + on other foreign servers would be rolled back. If the transaction on + PostgreSQL aborts transactions on all the foreign servers + are aborted too. + + diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index bd371fd..edd86cf 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -117,6 +117,7 @@ + diff --git a/doc/src/sgml/postgres-fdw.sgml b/doc/src/sgml/postgres-fdw.sgml index d83fc9e..3ce77a3 100644 --- a/doc/src/sgml/postgres-fdw.sgml +++ b/doc/src/sgml/postgres-fdw.sgml @@ -436,6 +436,42 @@ + + + Transaction Management Options + + + By default, if the transaction involves with multiple remote server, + each transaction on remote server is committed or aborted independently. + Some of transactions may fail to commit on remote server while other + transactions commit successfully. This may be overridden using + following option: + + + + + + two_phase_commit + + + This option controls whether postgres_fdw allows + to use two-phase-commit when transaction commits. This option can + only be sepcified for foreign servers, not per-table. + The default is false. + + + + If this option is enabled, postgres_fdw prepares + transaction on remote server and PostgreSQL + keeps track of the distributed transaction. + must be set more + than 1 on local server and + must set to more than 1 on remote server. + + + + + diff --git a/src/backend/access/rmgrdesc/Makefile b/src/backend/access/rmgrdesc/Makefile index 5514db1..6e23ec1 100644 --- a/src/backend/access/rmgrdesc/Makefile +++ b/src/backend/access/rmgrdesc/Makefile @@ -8,9 +8,10 @@ subdir = src/backend/access/rmgrdesc top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global -OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o genericdesc.o \ - gindesc.o gistdesc.o hashdesc.o heapdesc.o logicalmsgdesc.o \ - mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o seqdesc.o \ - smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o +OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o fdw_xactdesc.o \ + genericdesc.o gindesc.o gistdesc.o hashdesc.o heapdesc.o \ + logicalmsgdesc.o mxactdesc.o nbtdesc.o relmapdesc.o replorigindesc.o \ + seqdesc.o smgrdesc.o spgdesc.o standbydesc.o tblspcdesc.o xactdesc.o \ + xlogdesc.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/rmgrdesc/fdw_xactdesc.c b/src/backend/access/rmgrdesc/fdw_xactdesc.c new file mode 100644 index 0000000..fd4957c --- /dev/null +++ b/src/backend/access/rmgrdesc/fdw_xactdesc.c @@ -0,0 +1,69 @@ +/*------------------------------------------------------------------------- + * + * fdw_xactdesc.c + * PostgreSQL distributed transaction manager for foreign server. + * + * This module describes the WAL records for foreign transaction manager. + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/backend/access/transam/fdw_xactdesc.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/fdw_xact.h" +#include "access/xloginsert.h" +#include "lib/stringinfo.h" + +extern void +fdw_xact_desc(StringInfo buf, XLogReaderState *record) +{ + char *rec = XLogRecGetData(record); + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + if (info == XLOG_FDW_XACT_INSERT) + { + FDWXactOnDiskData *fdw_insert_xlog = (FDWXactOnDiskData *) rec; + + appendStringInfo(buf, "Foreign server oid: %u", fdw_insert_xlog->serverid); + appendStringInfo(buf, " user oid: %u", fdw_insert_xlog->userid); + appendStringInfo(buf, " database id: %u", fdw_insert_xlog->dboid); + appendStringInfo(buf, " local xid: %u", fdw_insert_xlog->local_xid); + /* TODO: This should be really interpreted by each FDW */ + + /* + * TODO: we also need to assess whether we want to add this + * information + */ + appendStringInfo(buf, " foreign transaction info: "); + appendStringInfo(buf, "%.*s", fdw_insert_xlog->fdw_xact_id_len, + fdw_insert_xlog->fdw_xact_id); + } + else + { + FdwRemoveXlogRec *fdw_remove_xlog = (FdwRemoveXlogRec *) rec; + + appendStringInfo(buf, "Foreign server oid: %u", fdw_remove_xlog->serverid); + appendStringInfo(buf, " user oid: %u", fdw_remove_xlog->userid); + appendStringInfo(buf, " database id: %u", fdw_remove_xlog->dbid); + appendStringInfo(buf, " local xid: %u", fdw_remove_xlog->xid); + } + +} + +extern const char * +fdw_xact_identify(uint8 info) +{ + switch (info & ~XLR_INFO_MASK) + { + case XLOG_FDW_XACT_INSERT: + return "NEW FOREIGN TRANSACTION"; + case XLOG_FDW_XACT_REMOVE: + return "REMOVE FOREIGN TRANSACTION"; + } + /* Keep compiler happy */ + return NULL; +} diff --git a/src/backend/access/rmgrdesc/xlogdesc.c b/src/backend/access/rmgrdesc/xlogdesc.c index f72f076..d35d8e3 100644 --- a/src/backend/access/rmgrdesc/xlogdesc.c +++ b/src/backend/access/rmgrdesc/xlogdesc.c @@ -112,14 +112,15 @@ xlog_desc(StringInfo buf, XLogReaderState *record) appendStringInfo(buf, "max_connections=%d max_worker_processes=%d " "max_prepared_xacts=%d max_locks_per_xact=%d " "wal_level=%s wal_log_hints=%s " - "track_commit_timestamp=%s", + "track_commit_timestamp=%s max_fdw_xacts=%d", xlrec.MaxConnections, xlrec.max_worker_processes, xlrec.max_prepared_xacts, xlrec.max_locks_per_xact, wal_level_str, xlrec.wal_log_hints ? "on" : "off", - xlrec.track_commit_timestamp ? "on" : "off"); + xlrec.track_commit_timestamp ? "on" : "off", + xlrec.max_prepared_foreign_xacts); } else if (info == XLOG_FPW_CHANGE) { diff --git a/src/backend/access/transam/Makefile b/src/backend/access/transam/Makefile index 16fbe47..dd7ee32 100644 --- a/src/backend/access/transam/Makefile +++ b/src/backend/access/transam/Makefile @@ -15,7 +15,7 @@ include $(top_builddir)/src/Makefile.global OBJS = clog.o commit_ts.o generic_xlog.o multixact.o parallel.o rmgr.o slru.o \ subtrans.o timeline.o transam.o twophase.o twophase_rmgr.o varsup.o \ xact.o xlog.o xlogarchive.o xlogfuncs.o \ - xloginsert.o xlogreader.o xlogutils.o + xloginsert.o xlogreader.o xlogutils.o fdw_xact.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/transam/fdw_xact.c b/src/backend/access/transam/fdw_xact.c new file mode 100644 index 0000000..2b35f5f --- /dev/null +++ b/src/backend/access/transam/fdw_xact.c @@ -0,0 +1,2205 @@ +/*------------------------------------------------------------------------- + * + * fdw_xact.c + * PostgreSQL distributed transaction manager for foreign server. + * + * This module manages the transactions involving foreign servers. + * + * Copyright (c) 2017, PostgreSQL Global Development Group + * + * src/backend/access/transam/fdw_xact.c + * + *------------------------------------------------------------------------- + */ +#include +#include +#include + +#include "postgres.h" + +#include "miscadmin.h" +#include "funcapi.h" + +#include "access/fdw_xact.h" +#include "access/htup_details.h" +#include "access/xact.h" +#include "access/xlog.h" +#include "access/xloginsert.h" +#include "access/xlogutils.h" +#include "catalog/pg_type.h" +#include "foreign/foreign.h" +#include "foreign/fdwapi.h" +#include "libpq/pqsignal.h" +#include "pg_trace.h" +#include "storage/fd.h" +#include "storage/ipc.h" +#include "storage/latch.h" +#include "storage/lock.h" +#include "storage/proc.h" +#include "storage/procarray.h" +#include "storage/shmem.h" +#include "utils/builtins.h" +#include "utils/memutils.h" +#include "utils/guc.h" +#include "utils/snapmgr.h" + +/* + * This comment summarises how the transaction manager handles transactions + * involving one or more foreign server/s. + * + * When an foreign data wrapper starts transaction on a foreign server, it is + * required to register the foreign server and user who initiated the + * transaction using function RegisterXactForeignServer(). A foreign server + * connection is identified by oid of foreign server and user. + * + * The commit is executed in two phases: + * First phase (executed during pre-commit processing) + * ----------- + * Transactions are prepared on all the foreign servers, which can participate + * in two-phase commit protocol. Transaction on other foreign servers are + * committed in the same phase. + * + * Second phase (executed during post-commit/abort processing) + * ------------ + * If first phase succeeds, foreign servers are requested to commit respective + * prepared transactions. If the first phase does not succeed because of any + * failure, the foreign servers are asked to rollback respective prepared + * transactions or abort the transactions if they are not prepared. + * + * Any network failure, server crash after preparing foreign transaction leaves + * that prepared transaction unresolved. During the first phase, before actually + * preparing the transactions, enough information is persisted to the disk and + * logs in order to resolve such transactions. + * + * During replay and replication FDWXactGlobal also holds information about + * active prepared foreign transaction that haven't been moved to disk yet. + * + * Replay of fdw_xact records happens by the following rules: + * + * * On PREPARE redo we add the foreign transaction to + * FDWXactGlobal->fdw_xacts. We set fdw_xact->inredo to true for + * such entries. + * + * * On Checkpoint redo we iterate through FDWXactGlobal->fdw_xacts. + * entries that have fdw_xact->inredo set and are behind the redo_horizon. + * We save them to disk and also set fdw_xact->ondisk to true. + * + * * On COMMIT/ABORT we delete the entry from FDWXactGlobal->fdw_xacts. + * If fdw_xact->ondisk is true, we delete the corresponding entry from + * the disk as well. + * + * * RecoverPreparedTransactions(), StnadbyReoverPreparedTransactions() and + * PrescanPreparedTransactions() have been modified to go through + * fdw_xact->inredo entries that have not made to disk yet. + */ + +/* Shared memory entry for a prepared or being prepared foreign transaction */ +typedef struct FDWXactData *FDWXact; + +/* Structure to bundle the foreign connection participating in transaction */ +typedef struct +{ + Oid serverid; + Oid userid; + Oid umid; + char *servername; + FDWXact fdw_xact; /* foreign prepared transaction entry in case + * prepared */ + bool two_phase_commit; /* Should use two phase commit + * protocol while committing + * transaction on this server, + * whenever necessary. */ + GetPrepareId_function get_prepare_id; + EndForeignTransaction_function end_foreign_xact; + PrepareForeignTransaction_function prepare_foreign_xact; + ResolvePreparedForeignTransaction_function resolve_prepared_foreign_xact; +} FDWConnection; + +/* List of foreign connections participating in the transaction */ +List *MyFDWConnections = NIL; + +/* + * By default we assume that all the foreign connections participating in this + * transaction can use two phase commit protocol. + */ +bool TwoPhaseReady = true; + +/* Record the server, userid participating in the transaction. */ +void +RegisterXactForeignServer(Oid serverid, Oid userid, bool two_phase_commit) +{ + FDWConnection *fdw_conn; + ListCell *lcell; + ForeignServer *foreign_server; + ForeignDataWrapper *fdw; + UserMapping *user_mapping; + FdwRoutine *fdw_routine; + MemoryContext old_context; + + TwoPhaseReady = TwoPhaseReady && two_phase_commit; + + /* Check if the entry already exists, if so, raise an error */ + foreach(lcell, MyFDWConnections) + { + fdw_conn = lfirst(lcell); + + if (fdw_conn->serverid == serverid && + fdw_conn->userid == userid) + ereport(ERROR, + (errmsg("attempt to start transction again on server %u user %u", + serverid, userid))); + } + + /* + * This list and its contents needs to be saved in the transaction context + * memory + */ + old_context = MemoryContextSwitchTo(TopTransactionContext); + /* Add this foreign connection to the list for transaction management */ + fdw_conn = (FDWConnection *) palloc(sizeof(FDWConnection)); + + /* Make sure that the FDW has at least a transaction handler */ + foreign_server = GetForeignServer(serverid); + fdw = GetForeignDataWrapper(foreign_server->fdwid); + fdw_routine = GetFdwRoutine(fdw->fdwhandler); + user_mapping = GetUserMapping(userid, serverid); + + if (!fdw_routine->EndForeignTransaction) + ereport(ERROR, + (errmsg("no function to end a foreign transaction provided for FDW %s", + fdw->fdwname))); + + if (two_phase_commit) + { + if (max_prepared_foreign_xacts == 0) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("prepread foreign transactions are disabled"), + errhint("Set max_prepared_foreign_transactions to a nonzero value."))); + + if (!fdw_routine->GetPrepareId) + ereport(ERROR, + (errmsg("no prepared transaction identifier providing function for FDW %s", + fdw->fdwname))); + + if (!fdw_routine->PrepareForeignTransaction) + ereport(ERROR, + (errmsg("no function provided for preparing foreign transaction for FDW %s", + fdw->fdwname))); + + if (!fdw_routine->ResolvePreparedForeignTransaction) + ereport(ERROR, + (errmsg("no function provided for resolving prepared foreign transaction for FDW %s", + fdw->fdwname))); + } + + fdw_conn->serverid = serverid; + fdw_conn->userid = userid; + fdw_conn->umid = user_mapping->umid; + + /* + * We may need following information at the end of a transaction, when the + * system caches are not available. So save it before hand. + */ + fdw_conn->servername = foreign_server->servername; + fdw_conn->get_prepare_id = fdw_routine->GetPrepareId; + fdw_conn->prepare_foreign_xact = fdw_routine->PrepareForeignTransaction; + fdw_conn->resolve_prepared_foreign_xact = fdw_routine->ResolvePreparedForeignTransaction; + fdw_conn->end_foreign_xact = fdw_routine->EndForeignTransaction; + fdw_conn->fdw_xact = NULL; + fdw_conn->two_phase_commit = two_phase_commit; + MyFDWConnections = lappend(MyFDWConnections, fdw_conn); + /* Revert back the context */ + MemoryContextSwitchTo(old_context); + + return; +} + +/* Prepared transaction identifier can be maximum 256 bytes long */ +#define MAX_FDW_XACT_ID_LEN 256 + +/* Enum to track the status of prepared foreign transaction */ +typedef enum +{ + FDW_XACT_PREPARING, /* foreign transaction is (being) prepared */ + FDW_XACT_COMMITTING_PREPARED, /* foreign prepared transaction is to + * be committed */ + FDW_XACT_ABORTING_PREPARED, /* foreign prepared transaction is to be + * aborted */ + FDW_XACT_RESOLVED /* Status used only by pg_fdw_xact_resolve(). + * It doesn't appear in the in-memory entry. */ +} FDWXactStatus; + +typedef struct FDWXactData +{ + FDWXact fx_next; /* Next free FDWXact entry */ + Oid dboid; /* database oid where to find foreign server + * and user mapping */ + TransactionId local_xid; /* XID of local transaction */ + Oid serverid; /* foreign server where transaction takes + * place */ + Oid userid; /* user who initiated the foreign transaction */ + Oid umid; /* user mapping id for connection key */ + FDWXactStatus status; /* The state of the foreign + * transaction. This doubles as the + * action to be taken on this entry. */ + + /* + * Note that we need to keep track of two LSNs for each FDWXact. We keep + * track of the start LSN because this is the address we must use to read + * state data back from WAL when committing a FDWXact. We keep track of + * the end LSN because that is the LSN we need to wait for prior to + * commit. + */ + XLogRecPtr fdw_xact_start_lsn; /* XLOG offset of inserting this entry + * start */ + XLogRecPtr fdw_xact_end_lsn; /* XLOG offset of inserting this entry + * end */ + + bool valid; /* Has the entry been complete and written to file? */ + BackendId locking_backend; /* Backend working on this entry */ + bool ondisk; /* TRUE if prepare state file is on disk */ + bool inredo; /* TRUE if entry was added via xlog_redo */ + int fdw_xact_id_len; /* Length of prepared transaction identifier */ + char fdw_xact_id[MAX_FDW_XACT_ID_LEN]; /* prepared transaction id */ +} FDWXactData; + +/* Directory where the foreign prepared transaction files will reside */ +#define FDW_XACTS_DIR "pg_fdw_xact" + +/* + * Name of foreign prepared transaction file is 8 bytes xid, 8 bytes foreign + * server oid and 8 bytes user oid separated by '_'. + */ +#define FDW_XACT_FILE_NAME_LEN (8 + 1 + 8 + 1 + 8) +#define FDWXactFilePath(path, xid, serverid, userid) \ + snprintf(path, MAXPGPATH, FDW_XACTS_DIR "/%08X_%08X_%08X", xid, \ + serverid, userid) + +/* Shared memory layout for maintaining foreign prepared transaction entries. */ +typedef struct +{ + /* Head of linked list of free FDWXactData structs */ + FDWXact freeFDWXacts; + + /* Number of valid FDW transaction entries */ + int numFDWXacts; + + /* Upto max_prepared_foreign_xacts entries in the array */ + FDWXact fdw_xacts[FLEXIBLE_ARRAY_MEMBER]; /* Variable length array */ +} FDWXactGlobalData; + +static void AtProcExit_FDWXact(int code, Datum arg); +static bool resolve_fdw_xact(FDWXact fdw_xact, + ResolvePreparedForeignTransaction_function prepared_foreign_xact_resolver); +static FDWXact insert_fdw_xact(Oid dboid, TransactionId xid, Oid serverid, Oid userid, + Oid umid, int fdw_xact_id_len, char *fdw_xact_id); +static void unlock_fdw_xact(FDWXact fdw_xact); +static void unlock_fdw_xact_entries(); +static void remove_fdw_xact(FDWXact fdw_xact); +static FDWXact register_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid, + Oid umid, int fdw_xact_info_len, char *fdw_xact_info); +static int GetFDWXactList(FDWXact * fdw_xacts); +static ResolvePreparedForeignTransaction_function get_prepared_foreign_xact_resolver(FDWXact fdw_xact); +static FDWXactOnDiskData *ReadFDWXactFile(TransactionId xid, Oid serverid, + Oid userid); +static void RemoveFDWXactFile(TransactionId xid, Oid serverid, Oid userid, + bool giveWarning); +static void RecreateFDWXactFile(TransactionId xid, Oid serverid, Oid userid, + void *content, int len); +static void XlogReadFDWXactData(XLogRecPtr lsn, char **buf, int *len); +static void prepare_foreign_transactions(void); +static FDWXact get_fdw_xact(TransactionId xid, Oid serverid, Oid userid); +bool search_fdw_xact(TransactionId xid, Oid dbid, Oid serverid, Oid userid, + List **qualifying_xacts); + +/* + * Maximum number of foreign prepared transaction entries at any given time + * GUC variable, change requires restart. + */ +int max_prepared_foreign_xacts = 0; + +/* Keep track of registering process exit call back. */ +static bool fdwXactExitRegistered = false; + +/* Pointer to the shared memory holding the foreign transactions data */ +static FDWXactGlobalData *FDWXactGlobal; + +/* foreign transaction entries locked by this backend */ +List *MyLockedFDWXacts = NIL; + +/* + * FDWXactShmemSize + * Calculates the size of shared memory allocated for maintaining foreign + * prepared transaction entries. + */ +extern Size +FDWXactShmemSize(void) +{ + Size size; + + /* Need the fixed struct, foreign transaction information array */ + size = offsetof(FDWXactGlobalData, fdw_xacts); + size = add_size(size, mul_size(max_prepared_foreign_xacts, + sizeof(FDWXact))); + size = MAXALIGN(size); + size = add_size(size, mul_size(max_prepared_foreign_xacts, + sizeof(FDWXactData))); + + return size; +} + +/* + * FDWXactShmemInit + * Initialization of shared memory for maintaining foreign prepared transaction + * entries. The shared memory layout is defined in definition of + * FDWXactGlobalData structure. + */ +extern void +FDWXactShmemInit(void) +{ + bool found; + + FDWXactGlobal = ShmemInitStruct("Foreign transactions table", + FDWXactShmemSize(), + &found); + if (!IsUnderPostmaster) + { + FDWXact fdw_xacts; + int cnt; + + Assert(!found); + FDWXactGlobal->freeFDWXacts = NULL; + FDWXactGlobal->numFDWXacts = 0; + + /* Initialise the linked list of free FDW transactions */ + fdw_xacts = (FDWXact) + ((char *) FDWXactGlobal + + MAXALIGN(offsetof(FDWXactGlobalData, fdw_xacts) + + sizeof(FDWXact) * max_prepared_foreign_xacts)); + for (cnt = 0; cnt < max_prepared_foreign_xacts; cnt++) + { + fdw_xacts[cnt].fx_next = FDWXactGlobal->freeFDWXacts; + FDWXactGlobal->freeFDWXacts = &fdw_xacts[cnt]; + } + } + else + { + Assert(FDWXactGlobal); + Assert(found); + } +} + +/* + * PreCommit_FDWXacts + * + * The function is responsible for pre-commit processing on foreign connections. + * Basically the foreign transactions are prepared on the foreign servers which + * can execute two-phase-commit protocol. But in case of where only one server + * that can execute two-phase-commit protocol is involved with transaction and + * no changes is made on local data then we don't need to two-phase-commit protocol, + * so try to commit transaction on the server. Those will be aborted or committed + * after the current transaction has been aborted or committed resp. We try to + * commit transactions on rest of the foreign servers now. For these foreign + * servers it is possible that some transactions commit even if the local + * transaction aborts. + */ +void +PreCommit_FDWXacts(void) +{ + ListCell *cur; + ListCell *prev; + ListCell *next; + + /* If there are no foreign servers involved, we have no business here */ + if (list_length(MyFDWConnections) < 1) + return; + + /* + * Try committing transactions on the foreign servers, which can not + * execute two-phase-commit protocol. + */ + for (cur = list_head(MyFDWConnections), prev = NULL; cur; cur = next) + { + FDWConnection *fdw_conn = lfirst(cur); + + next = lnext(cur); + + if (!fdw_conn->two_phase_commit) + { + /* + * The FDW has to make sure that the connection opened to the + * foreign server is out of transaction. Even if the handler + * function returns failure statue, there's hardly anything to do. + */ + if (!fdw_conn->end_foreign_xact(fdw_conn->serverid, fdw_conn->userid, + fdw_conn->umid, true)) + elog(WARNING, "could not commit transaction on server %s", + fdw_conn->servername); + + /* The connection is no more part of this transaction, forget it */ + MyFDWConnections = list_delete_cell(MyFDWConnections, cur, prev); + } + else + prev = cur; + } + + /* + * Here foreign servers that can not execute two-phase-commit protocol + * already commit the transaction and MyFDWConnections has only foreign + * servers that can execute two-phase-commit protocol. We don't need to + * use two-phase-commit protocol if there is only one foreign server that + * that can execute two-phase-commit and didn't write no local node. + */ + if ((list_length(MyFDWConnections) > 1) || + (list_length(MyFDWConnections) == 1 && XactWriteLocalNode)) + { + /* + * Prepare the transactions on the all foreign servers, which can + * execute two-phase-commit protocol. + */ + prepare_foreign_transactions(); + } + else if (list_length(MyFDWConnections) == 1) + { + FDWConnection *fdw_conn = lfirst(list_head(MyFDWConnections)); + + /* + * We don't need to use two-phase commit protocol only one server + * remaining even if this server can execute two-phase-commit + * protocol. + */ + if (!fdw_conn->end_foreign_xact(fdw_conn->serverid, fdw_conn->userid, + fdw_conn->umid, true)) + elog(WARNING, "could not commit transaction on server %s", + fdw_conn->servername); + + /* MyFDWConnections should be cleared here */ + MyFDWConnections = list_delete_cell(MyFDWConnections, cur, prev); + } +} + +/* + * prepare_foreign_transactions + * + * Prepare transactions on the foreign servers which can execute two phase + * commit protocol. Rest of the foreign servers are ignored. + */ +static void +prepare_foreign_transactions(void) +{ + ListCell *lcell; + + /* + * Loop over the foreign connections + */ + foreach(lcell, MyFDWConnections) + { + FDWConnection *fdw_conn = (FDWConnection *) lfirst(lcell); + char *fdw_xact_id; + int fdw_xact_id_len; + FDWXact fdw_xact; + + if (!fdw_conn->two_phase_commit) + continue; + + Assert(fdw_conn->get_prepare_id); + fdw_xact_id = fdw_conn->get_prepare_id(fdw_conn->serverid, + fdw_conn->userid, + &fdw_xact_id_len); + + /* + * Register the foreign transaction with the identifier used to + * prepare it on the foreign server. Registration persists this + * information to the disk and logs (that way relaying it on standby). + * Thus in case we loose connectivity to the foreign server or crash + * ourselves, we will remember that we have prepared transaction on + * the foreign server and try to resolve it when connectivity is + * restored or after crash recovery. + * + * If we crash after persisting the information but before preparing + * the transaction on the foreign server, we will try to resolve a + * never-prepared transaction, and get an error. This is fine as long + * as the FDW provides us unique prepared transaction identifiers. + * + * If we prepare the transaction on the foreign server before + * persisting the information to the disk and crash in-between these + * two steps, we will forget that we prepared the transaction on the + * foreign server and will not be able to resolve it after the crash. + * Hence persist first then prepare. + */ + fdw_xact = register_fdw_xact(MyDatabaseId, GetTopTransactionId(), + fdw_conn->serverid, fdw_conn->userid, + fdw_conn->umid, fdw_xact_id_len, + fdw_xact_id); + + /* + * Between register_fdw_xact call above till this backend hears back + * from foreign server, the backend may abort the local transaction + * (say, because of a signal). During abort processing, it will send + * an ABORT message to the foreign server. If the foreign server has + * not prepared the transaction, the message will succeed. If the + * foreign server has prepared transaction, it will throw an error, + * which we will ignore and the prepared foreign transaction will be + * resolved by the foreign transaction resolver. + */ + if (!fdw_conn->prepare_foreign_xact(fdw_conn->serverid, fdw_conn->userid, + fdw_conn->umid, fdw_xact_id_len, + fdw_xact_id)) + { + /* + * An error occurred, and we didn't prepare the transaction. + * Delete the entry from foreign transaction table. Raise an + * error, so that the local server knows that one of the foreign + * server has failed to prepare the transaction. + * + * XXX : FDW is expected to print the error as a warning and then + * we raise actual error here. But instead, we should pull the + * error text from FDW and add it here in the message or as a + * context or a hint. + */ + remove_fdw_xact(fdw_xact); + + /* + * Delete the connection, since it doesn't require any further + * processing. This deletion will invalidate current cell pointer, + * but that is fine since we will not use that pointer because the + * subsequent ereport will get us out of this loop. + */ + MyFDWConnections = list_delete_ptr(MyFDWConnections, fdw_conn); + ereport(ERROR, + (errmsg("can not prepare transaction on foreign server %s", + fdw_conn->servername))); + } + + /* Prepare succeeded, remember it in the connection */ + fdw_conn->fdw_xact = fdw_xact; + } + return; +} + +/* + * register_fdw_xact + * + * This function is used to create new foreign transaction entry before an FDW + * executes the first phase of two-phase commit. The function adds the entry to + * WAL and will be persisted to the disk under pg_fdw_xact directory when checkpoint. + */ +static FDWXact +register_fdw_xact(Oid dbid, TransactionId xid, Oid serverid, Oid userid, + Oid umid, int fdw_xact_id_len, char *fdw_xact_id) +{ + FDWXact fdw_xact; + FDWXactOnDiskData *fdw_xact_file_data; + int data_len; + + /* Enter the foreign transaction in the shared memory structure */ + LWLockAcquire(FDWXactLock, LW_EXCLUSIVE); + fdw_xact = insert_fdw_xact(dbid, xid, serverid, userid, umid, + fdw_xact_id_len, fdw_xact_id); + fdw_xact->status = FDW_XACT_PREPARING; + fdw_xact->locking_backend = MyBackendId; + LWLockRelease(FDWXactLock); + + /* Remember that we have locked this entry. */ + MyLockedFDWXacts = lappend(MyLockedFDWXacts, fdw_xact); + + /* + * Prepare to write the entry to a file. Also add xlog entry. The contents + * of the xlog record are same as what is written to the file. + */ + data_len = offsetof(FDWXactOnDiskData, fdw_xact_id); + data_len = data_len + fdw_xact->fdw_xact_id_len; + data_len = MAXALIGN(data_len); + fdw_xact_file_data = (FDWXactOnDiskData *) palloc0(data_len); + fdw_xact_file_data->dboid = fdw_xact->dboid; + fdw_xact_file_data->local_xid = fdw_xact->local_xid; + fdw_xact_file_data->serverid = fdw_xact->serverid; + fdw_xact_file_data->userid = fdw_xact->userid; + fdw_xact_file_data->umid = fdw_xact->umid; + fdw_xact_file_data->fdw_xact_id_len = fdw_xact->fdw_xact_id_len; + memcpy(fdw_xact_file_data->fdw_xact_id, fdw_xact->fdw_xact_id, + fdw_xact->fdw_xact_id_len); + + START_CRIT_SECTION(); + + /* Add the entry in the xlog and save LSN for checkpointer */ + XLogBeginInsert(); + XLogRegisterData((char *) fdw_xact_file_data, data_len); + fdw_xact->fdw_xact_end_lsn = XLogInsert(RM_FDW_XACT_ID, XLOG_FDW_XACT_INSERT); + XLogFlush(fdw_xact->fdw_xact_end_lsn); + + /* Store record's start location to read that later on CheckPoint */ + fdw_xact->fdw_xact_start_lsn = ProcLastRecPtr; + + /* File is written completely, checkpoint can proceed with syncing */ + fdw_xact->valid = true; + + END_CRIT_SECTION(); + + pfree(fdw_xact_file_data); + return fdw_xact; +} + +/* + * insert_fdw_xact + * + * Insert a new entry for a given foreign transaction identified by transaction + * id, foreign server and user mapping, in the shared memory. Caller must hold + * FDWXactLock in exclusive mode. + * + * If the entry already exists, the function raises an error. + */ +static FDWXact +insert_fdw_xact(Oid dboid, TransactionId xid, Oid serverid, Oid userid, Oid umid, + int fdw_xact_id_len, char *fdw_xact_id) +{ + int i; + FDWXact fdw_xact; + + if (!fdwXactExitRegistered) + { + before_shmem_exit(AtProcExit_FDWXact, 0); + fdwXactExitRegistered = true; + } + + if (fdw_xact_id_len > MAX_FDW_XACT_ID_LEN) + elog(ERROR, "foreign transaction identifier longer (%d) than allowed (%d)", + fdw_xact_id_len, MAX_FDW_XACT_ID_LEN); + + /* Check for duplicating foreign transaction entry */ + for (i = 0; i < FDWXactGlobal->numFDWXacts; i++) + { + fdw_xact = FDWXactGlobal->fdw_xacts[i]; + if (fdw_xact->local_xid == xid && + fdw_xact->serverid == serverid && + fdw_xact->userid == userid) + elog(ERROR, "duplicate entry for foreign transaction with transaction id %u, serverid %u, userid %u found", + xid, serverid, userid); + } + + /* + * Get the next free foreign transaction entry. Raise error if there are + * none left. + */ + if (!FDWXactGlobal->freeFDWXacts) + { + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("maximum number of foreign transactions reached"), + errhint("Increase max_prepared_foreign_transactions (currently %d).", + max_prepared_foreign_xacts))); + } + + fdw_xact = FDWXactGlobal->freeFDWXacts; + FDWXactGlobal->freeFDWXacts = fdw_xact->fx_next; + + /* Insert the entry to active array */ + Assert(FDWXactGlobal->numFDWXacts < max_prepared_foreign_xacts); + FDWXactGlobal->fdw_xacts[FDWXactGlobal->numFDWXacts++] = fdw_xact; + + /* Stamp the entry with backend id before releasing the LWLock */ + fdw_xact->locking_backend = InvalidBackendId; + fdw_xact->dboid = dboid; + fdw_xact->local_xid = xid; + fdw_xact->serverid = serverid; + fdw_xact->userid = userid; + fdw_xact->umid = umid; + fdw_xact->fdw_xact_start_lsn = InvalidXLogRecPtr; + fdw_xact->fdw_xact_end_lsn = InvalidXLogRecPtr; + fdw_xact->valid = false; + fdw_xact->ondisk = false; + fdw_xact->inredo = false; + fdw_xact->fdw_xact_id_len = fdw_xact_id_len; + memcpy(fdw_xact->fdw_xact_id, fdw_xact_id, fdw_xact_id_len); + + return fdw_xact; +} + +/* + * remove_fdw_xact + * + * Removes the foreign prepared transaction entry from shared memory, disk and + * logs about the removal in WAL. + */ +static void +remove_fdw_xact(FDWXact fdw_xact) +{ + int cnt; + + LWLockAcquire(FDWXactLock, LW_EXCLUSIVE); + /* Search the slot where this entry resided */ + for (cnt = 0; cnt < FDWXactGlobal->numFDWXacts; cnt++) + { + if (FDWXactGlobal->fdw_xacts[cnt] == fdw_xact) + { + /* Remove the entry from active array */ + FDWXactGlobal->numFDWXacts--; + FDWXactGlobal->fdw_xacts[cnt] = FDWXactGlobal->fdw_xacts[FDWXactGlobal->numFDWXacts]; + + /* Put it back into free list */ + fdw_xact->fx_next = FDWXactGlobal->freeFDWXacts; + FDWXactGlobal->freeFDWXacts = fdw_xact; + + /* Unlock the entry */ + fdw_xact->locking_backend = InvalidBackendId; + MyLockedFDWXacts = list_delete_ptr(MyLockedFDWXacts, fdw_xact); + + LWLockRelease(FDWXactLock); + + if (!RecoveryInProgress()) + { + FdwRemoveXlogRec fdw_remove_xlog; + XLogRecPtr recptr; + + /* Fill up the log record before releasing the entry */ + fdw_remove_xlog.serverid = fdw_xact->serverid; + fdw_remove_xlog.dbid = fdw_xact->dboid; + fdw_remove_xlog.xid = fdw_xact->local_xid; + fdw_remove_xlog.userid = fdw_xact->userid; + + START_CRIT_SECTION(); + + /* + * Log that we are removing the foreign transaction entry and + * remove the file from the disk as well. + */ + XLogBeginInsert(); + XLogRegisterData((char *) &fdw_remove_xlog, sizeof(fdw_remove_xlog)); + recptr = XLogInsert(RM_FDW_XACT_ID, XLOG_FDW_XACT_REMOVE); + XLogFlush(recptr); + + END_CRIT_SECTION(); + } + + /* Remove the file from the disk if exists. */ + if (fdw_xact->ondisk) + RemoveFDWXactFile(fdw_xact->local_xid, fdw_xact->serverid, + fdw_xact->userid, true); + return; + } + } + LWLockRelease(FDWXactLock); + + /* We did not find the given entry in global array */ + elog(ERROR, "failed to find %p in FDWXactGlobal array", fdw_xact); +} + +/* + * unlock_fdw_xact + * + * Unlock the foreign transaction entry by wiping out the locking_backend and + * removing it from the backend's list of foreign transaction. + */ +static void +unlock_fdw_xact(FDWXact fdw_xact) +{ + /* Only the backend holding the lock is allowed to unlock */ + Assert(fdw_xact->locking_backend == MyBackendId); + + /* + * First set the locking backend as invalid, and then remove it from the + * list of locked foreign transactions, under the LW lock. If we reverse + * the order and process exits in-between those two, we will be left an + * entry locked by this backend, which gets unlocked only at the server + * restart. + */ + + LWLockAcquire(FDWXactLock, LW_EXCLUSIVE); + fdw_xact->locking_backend = InvalidBackendId; + MyLockedFDWXacts = list_delete_ptr(MyLockedFDWXacts, fdw_xact); + LWLockRelease(FDWXactLock); +} + +/* + * unlock_fdw_xact_entries + * + * Unlock the foreign transaction entries locked by this backend. + */ +static void +unlock_fdw_xact_entries() +{ + while (MyLockedFDWXacts) + { + FDWXact fdw_xact = (FDWXact) linitial(MyLockedFDWXacts); + + unlock_fdw_xact(fdw_xact); + } +} + +/* + * AtProcExit_FDWXact + * + * When the process exits, unlock the entries it held. + */ +static void +AtProcExit_FDWXact(int code, Datum arg) +{ + unlock_fdw_xact_entries(); +} + +/* + * AtEOXact_FDWXacts + * + * The function executes phase 2 of two-phase commit protocol. + * At the end of transaction perform following actions + * 1. Mark the entries locked by this backend as ABORTING or COMMITTING + * according the result of transaction. + * 2. Try to commit or abort the transactions on foreign servers. If that + * succeeds, remove them from foreign transaction entries, otherwise unlock + * them. + */ +extern void +AtEOXact_FDWXacts(bool is_commit) +{ + ListCell *lcell; + + foreach(lcell, MyFDWConnections) + { + FDWConnection *fdw_conn = lfirst(lcell); + + /* Commit/abort prepared foreign transactions */ + if (fdw_conn->fdw_xact) + { + FDWXact fdw_xact = fdw_conn->fdw_xact; + + fdw_xact->status = (is_commit ? + FDW_XACT_COMMITTING_PREPARED : + FDW_XACT_ABORTING_PREPARED); + + /* + * Try aborting or committing the transaction on the foreign + * server + */ + if (!resolve_fdw_xact(fdw_xact, fdw_conn->resolve_prepared_foreign_xact)) + { + /* + * The transaction was not resolved on the foreign server, + * unlock it, so that someone else can take care of it. + */ + unlock_fdw_xact(fdw_xact); + } + } + else + { + /* + * On servers where two phase commit protocol could not be + * executed we have tried to commit the transactions during + * pre-commit phase. Any remaining transactions need to be + * aborted. + */ + Assert(!is_commit); + + /* + * The FDW has to make sure that the connection opened to the + * foreign server is out of transaction. Even if the handler + * function returns failure statue, there's hardly anything to do. + */ + if (!fdw_conn->end_foreign_xact(fdw_conn->serverid, fdw_conn->userid, + fdw_conn->umid, is_commit)) + elog(WARNING, "could not %s transaction on server %s", + is_commit ? "commit" : "abort", + fdw_conn->servername); + + } + } + + /* + * Unlock any locked foreign transactions. Resolver might lock the + * entries, and may not be able to unlock them if aborted in-between. In + * any case, there is no reason for a foreign transaction entry to be + * locked after the transaction which locked it has ended. + */ + unlock_fdw_xact_entries(); + + /* + * Reset the list of registered connections. Since the memory for the list + * and its nodes comes from transaction memory context, it will be freed + * after this call. + */ + MyFDWConnections = NIL; + /* Set TwoPhaseReady to its default value */ + TwoPhaseReady = true; +} + +/* + * AtPrepare_FDWXacts + * + * The function is called while preparing a transaction. If there are foreign + * servers involved in the transaction, this function prepares transactions + * on those servers. + */ +extern void +AtPrepare_FDWXacts(void) +{ + /* If there are no foreign servers involved, we have no business here */ + if (list_length(MyFDWConnections) < 1) + return; + + /* + * All foreign servers participating in a transaction to be prepared + * should be two phase compliant. + */ + if (!TwoPhaseReady) + ereport(ERROR, + (errcode(ERRCODE_T_R_INTEGRITY_CONSTRAINT_VIOLATION), + errmsg("can not prepare the transaction because some foreign servers involved in transaction can not prepare the transaction"))); + + /* Prepare transactions on participating foreign servers. */ + prepare_foreign_transactions(); + + /* + * Unlock the foreign transaction entries so COMMIT/ROLLBACK PREPARED from + * some other backend will be able to lock those if required. + */ + unlock_fdw_xact_entries(); + + /* + * Reset the list of registered connections. Since the memory for the list + * and its nodes comes from transaction memory context, it will be freed + * after this call. + */ + MyFDWConnections = NIL; + + /* Set TwoPhaseReady to its default value */ + TwoPhaseReady = true; +} + +/* + * FDWXactTwoPhaseFinish + * + * This function is called as part of the COMMIT/ROLLBACK PREPARED command to + * commit/rollback the foreign transactions prepared as part of the local + * prepared transaction. The function looks for the foreign transaction entries + * with local_xid equal to xid of the prepared transaction and tries to resolve them. + */ +extern void +FDWXactTwoPhaseFinish(bool isCommit, TransactionId xid) +{ + List *entries_to_resolve; + + FDWXactStatus status = isCommit ? FDW_XACT_COMMITTING_PREPARED : + FDW_XACT_ABORTING_PREPARED; + + /* + * Get all the entries belonging to the given transaction id locked. If + * foreign transaction resolver is running, it might lock entries to check + * whether they can be resolved. The search function will skip such + * entries. The resolver will resolve them at a later point of time. + */ + search_fdw_xact(xid, InvalidOid, InvalidOid, InvalidOid, &entries_to_resolve); + + /* Try resolving the foreign transactions */ + while (entries_to_resolve) + { + FDWXact fdw_xact = linitial(entries_to_resolve); + + entries_to_resolve = list_delete_first(entries_to_resolve); + fdw_xact->status = status; + + /* + * Resolve the foreign transaction. If resolution is not successful, + * unlock the entry so that someone else can pick it up. + */ + if (!resolve_fdw_xact(fdw_xact, + get_prepared_foreign_xact_resolver(fdw_xact))) + unlock_fdw_xact(fdw_xact); + } +} + +/* + * get_prepared_foreign_xact_resolver + */ +static ResolvePreparedForeignTransaction_function +get_prepared_foreign_xact_resolver(FDWXact fdw_xact) +{ + ForeignServer *foreign_server; + ForeignDataWrapper *fdw; + FdwRoutine *fdw_routine; + + foreign_server = GetForeignServer(fdw_xact->serverid); + fdw = GetForeignDataWrapper(foreign_server->fdwid); + fdw_routine = GetFdwRoutine(fdw->fdwhandler); + if (!fdw_routine->ResolvePreparedForeignTransaction) + elog(ERROR, "no foreign transaction resolver routine provided for FDW %s", + fdw->fdwname); + + return fdw_routine->ResolvePreparedForeignTransaction; +} + +/* + * resolve_fdw_xact + * + * Resolve the foreign transaction using the foreign data wrapper's transaction + * handler routine. + * If the resolution is successful, remove the foreign transaction entry from + * the shared memory and also remove the corresponding on-disk file. + */ +static bool +resolve_fdw_xact(FDWXact fdw_xact, + ResolvePreparedForeignTransaction_function fdw_xact_handler) +{ + bool resolved; + bool is_commit; + + Assert(fdw_xact->status == FDW_XACT_COMMITTING_PREPARED || + fdw_xact->status == FDW_XACT_ABORTING_PREPARED); + + is_commit = (fdw_xact->status == FDW_XACT_COMMITTING_PREPARED) ? + true : false; + + resolved = fdw_xact_handler(fdw_xact->serverid, fdw_xact->userid, + fdw_xact->umid, is_commit, + fdw_xact->fdw_xact_id_len, + fdw_xact->fdw_xact_id); + + /* If we succeeded in resolving the transaction, remove the entry */ + if (resolved) + remove_fdw_xact(fdw_xact); + + return resolved; +} + +/* + * Get foreign transaction entry from FDWXactGlobal->fdw_xacts. Return NULL + * if foreign transacgiven does not exist. + */ +static FDWXact +get_fdw_xact(TransactionId xid, Oid serverid, Oid userid) +{ + int i; + FDWXact fdw_xact; + + LWLockAcquire(FDWXactLock, LW_SHARED); + + for (i = 0; i < FDWXactGlobal->numFDWXacts; i++) + { + fdw_xact = FDWXactGlobal->fdw_xacts[i]; + + if (fdw_xact->local_xid == xid && + fdw_xact->serverid == serverid && + fdw_xact->userid == userid) + { + LWLockRelease(FDWXactLock); + return fdw_xact; + } + } + + LWLockRelease(FDWXactLock); + return NULL; +} + +/* + * fdw_xact_exists + * Returns true if there exists at least one prepared foreign transaction which + * matches criteria. This function is wrapper around search_fdw_xact. Check that + * function's prologue for details. + */ +bool +fdw_xact_exists(TransactionId xid, Oid dbid, Oid serverid, Oid userid) +{ + return search_fdw_xact(xid, dbid, serverid, userid, NULL); +} + +/* + * search_fdw_xact + * Return true if there exists at least one prepared foreign transaction + * entry with given criteria. The criteria is defined by arguments with + * valid values for respective datatypes. + * + * The table below explains the same + * xid | dbid | serverid | userid | search for entry with + * invalid | invalid | invalid | invalid | nothing + * invalid | invalid | invalid | valid | given userid + * invalid | invalid | valid | invalid | given serverid + * invalid | invalid | valid | valid | given serverid and userid + * invalid | valid | invalid | invalid | given dbid + * invalid | valid | invalid | valid | given dbid and userid + * invalid | valid | valid | invalid | given dbid and serverid + * invalid | valid | valid | valid | given dbid, serveroid and userid + * valid | invalid | invalid | invalid | given xid + * valid | invalid | invalid | valid | given xid and userid + * valid | invalid | valid | invalid | given xid, serverid + * valid | invalid | valid | valid | given xid, serverid, userid + * valid | valid | invalid | invalid | given xid and dbid + * valid | valid | invalid | valid | given xid, dbid and userid + * valid | valid | valid | invalid | given xid, dbid, serverid + * valid | valid | valid | valid | given xid, dbid, serverid, userid + * + * When the criteria is void (all arguments invalid) the + * function returns true, since any entry would match the criteria. + * + * If qualifying_fdw_xacts is not NULL, the qualifying entries are locked and + * returned in a linked list. Any entry which is already locked is ignored. If + * all the qualifying entries are locked, nothing will be returned in the list + * but returned value will be true. + */ +bool +search_fdw_xact(TransactionId xid, Oid dbid, Oid serverid, Oid userid, + List **qualifying_xacts) +{ + int cnt; + LWLockMode lock_mode; + + /* Return value if a qualifying entry exists */ + bool entry_exists = false; + + if (qualifying_xacts) + { + *qualifying_xacts = NIL; + /* The caller expects us to lock entries */ + lock_mode = LW_EXCLUSIVE; + } + else + lock_mode = LW_SHARED; + + LWLockAcquire(FDWXactLock, lock_mode); + for (cnt = 0; cnt < FDWXactGlobal->numFDWXacts; cnt++) + { + FDWXact fdw_xact = FDWXactGlobal->fdw_xacts[cnt]; + bool entry_matches = true; + + /* xid */ + if (xid != InvalidTransactionId && xid != fdw_xact->local_xid) + entry_matches = false; + + /* dbid */ + if (OidIsValid(dbid) && fdw_xact->dboid != dbid) + entry_matches = false; + + /* serverid */ + if (OidIsValid(serverid) && serverid != fdw_xact->serverid) + entry_matches = false; + + /* userid */ + if (OidIsValid(userid) && fdw_xact->userid != userid) + entry_matches = false; + + if (entry_matches) + { + entry_exists = true; + if (qualifying_xacts) + { + /* + * User has requested list of qualifying entries. If the + * matching entry is not locked, lock it and add to the list. + * If the entry is locked by some other backend, ignore it. + */ + if (fdw_xact->locking_backend == InvalidBackendId) + { + MemoryContext oldcontext; + + fdw_xact->locking_backend = MyBackendId; + + /* + * The list and its members may be required at the end of + * the transaction + */ + oldcontext = MemoryContextSwitchTo(TopTransactionContext); + MyLockedFDWXacts = lappend(MyLockedFDWXacts, fdw_xact); + MemoryContextSwitchTo(oldcontext); + } + else if (fdw_xact->locking_backend != MyBackendId) + continue; + + *qualifying_xacts = lappend(*qualifying_xacts, fdw_xact); + } + else + { + /* + * User wants to check the existence, and we have found one + * matching entry. No need to check other entries. + */ + break; + } + } + } + + LWLockRelease(FDWXactLock); + + return entry_exists; +} + +/* + * fdw_xact_redo + * Apply the redo log for a foreign transaction. + */ +extern void +fdw_xact_redo(XLogReaderState *record) +{ + char *rec = XLogRecGetData(record); + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + if (info == XLOG_FDW_XACT_INSERT) + FDWXactRedoAdd(record); + else if (info == XLOG_FDW_XACT_REMOVE) + { + FdwRemoveXlogRec *fdw_remove_xlog = (FdwRemoveXlogRec *) rec; + + /* Delete FDWXact entry and file if exists */ + FDWXactRedoRemove(fdw_remove_xlog->xid, fdw_remove_xlog->serverid, + fdw_remove_xlog->userid); + } + else + elog(ERROR, "invalid log type %d in foreign transction log record", info); + + return; +} + +/* + * CheckPointFDWXact + * + * Function syncs the foreign transaction files created between the two + * checkpoints. The foreign transaction entries and hence the corresponding + * files are expected to be very short-lived. By executing this function at the + * end, we might have lesser files to fsync, thus reducing some I/O. This is + * similar to CheckPointTwoPhase(). + * + * In order to avoid disk I/O while holding a light weight lock, the function + * first collects the files which need to be synced under FDWXactLock and then + * syncs them after releasing the lock. This approach creates a race condition: + * after releasing the lock, and before syncing a file, the corresponding + * foreign transaction entry and hence the file might get removed. The function + * checks whether that's true and ignores the error if so. + */ +void +CheckPointFDWXact(XLogRecPtr redo_horizon) +{ + int cnt; + int serialized_fdw_xacts = 0; + + /* Quick get-away, before taking lock */ + if (max_prepared_foreign_xacts <= 0) + return; + + TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_START(); + + LWLockAcquire(FDWXactLock, LW_SHARED); + + /* Another quick, before we allocate memory */ + if (FDWXactGlobal->numFDWXacts <= 0) + { + LWLockRelease(FDWXactLock); + return; + } + + /* + * We are expecting there to be zero FDWXact that need to be copied to + * disk, so we perform all I/O while holding FDWXactLock for simplicity. + * This presents any new foreign xacts from preparing while this occurs, + * which shouldn't be a problem since the presence fo long-lived prepared + * foreign xacts indicated the transaction manager isn't active. + * + * it's also possible to move I/O out of the lock, but on every error we + * should check whether somebody committed our transaction in different + * backend. Let's leave this optimisation for future, if somebody will + * spot that this place cause bottleneck. + * + * Note that it isn't possible for there to be a FDWXact with a + * fdw_xact_end_lsn set prior to the last checkpoint yet is marked + * invalid, because of the efforts with delayChkpt. + */ + for (cnt = 0; cnt < FDWXactGlobal->numFDWXacts; cnt++) + { + FDWXact fdw_xact = FDWXactGlobal->fdw_xacts[cnt]; + + if ((fdw_xact->valid || fdw_xact->inredo) && + !fdw_xact->ondisk && + fdw_xact->fdw_xact_end_lsn <= redo_horizon) + { + char *buf; + int len; + + XlogReadFDWXactData(fdw_xact->fdw_xact_start_lsn, &buf, &len); + RecreateFDWXactFile(fdw_xact->local_xid, fdw_xact->serverid, + fdw_xact->userid, buf, len); + fdw_xact->ondisk = true; + serialized_fdw_xacts++; + pfree(buf); + } + } + + LWLockRelease(FDWXactLock); + + TRACE_POSTGRESQL_FDWXACT_CHECKPOINT_DONE(); + + if (log_checkpoints && serialized_fdw_xacts > 0) + ereport(LOG, + (errmsg_plural("%u foreign transaction state file was written " + "for long-running prepared transactions", + "%u foreign transaction state files were written " + "for long-running prepared transactions", + serialized_fdw_xacts, + serialized_fdw_xacts))); +} + +/* + * Reads foreign trasasction data from xlog. During checkpoint this data will + * be moved to fdwxact files and ReadFDWXactFile should be used instead. + * + * Note clearly that this function accesses WAL during normal operation, similarly + * to the way WALSender or Logical Decoding would do. It does not run during + * crash recovery or standby processing. + */ +static void +XlogReadFDWXactData(XLogRecPtr lsn, char **buf, int *len) +{ + XLogRecord *record; + XLogReaderState *xlogreader; + char *errormsg; + + xlogreader = XLogReaderAllocate(&read_local_xlog_page, NULL); + if (!xlogreader) + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"), + errdetail("Failed while allocating an XLog reading processor."))); + + record = XLogReadRecord(xlogreader, lsn, &errormsg); + + if (record == NULL) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not read foreign transaction state from xlog at %X/%X", + (uint32) (lsn >> 32), + (uint32) lsn))); + + if (XLogRecGetRmid(xlogreader) != RM_FDW_XACT_ID || + (XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK) != XLOG_FDW_XACT_INSERT) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("expected foreign transaction state data is not present in xlog at %X/%X", + (uint32) (lsn >> 32), + (uint32) lsn))); + + if (len != NULL) + *len = XLogRecGetDataLen(xlogreader); + + *buf = palloc(sizeof(char) * XLogRecGetDataLen(xlogreader)); + memcpy(*buf, XLogRecGetData(xlogreader), sizeof(char) * XLogRecGetDataLen(xlogreader)); + + XLogReaderFree(xlogreader); +} + +/* + * Recreates a foreign transaction state file. This is used in WAL replay and + * during checkpoint creation. + * + * Note: content and len don't include CRC. + */ +void +RecreateFDWXactFile(TransactionId xid, Oid serverid, Oid userid, + void *content, int len) +{ + char path[MAXPGPATH]; + pg_crc32c fdw_xact_crc; + pg_crc32c bogus_crc; + int fd; + + /* Recompute CRC */ + INIT_CRC32C(fdw_xact_crc); + COMP_CRC32C(fdw_xact_crc, content, len); + + FDWXactFilePath(path, xid, serverid, userid); + + fd = OpenTransientFile(path, O_CREAT | O_TRUNC | O_WRONLY | PG_BINARY, + S_IRUSR | S_IWUSR); + + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not recreate foreign transaction state file \"%s\": %m", + path))); + + if (write(fd, content, len) != len) + { + CloseTransientFile(fd); + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write foreign transcation state file: %m"))); + } + FIN_CRC32C(fdw_xact_crc); + + /* + * Write a deliberately bogus CRC to the state file; this is just paranoia + * to catch the case where four more bytes will run us out of disk space. + */ + bogus_crc = ~fdw_xact_crc; + if ((write(fd, &bogus_crc, sizeof(pg_crc32c))) != sizeof(pg_crc32c)) + { + CloseTransientFile(fd); + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write foreing transaction state file: %m"))); + } + /* Back up to prepare for rewriting the CRC */ + if (lseek(fd, -((off_t) sizeof(pg_crc32c)), SEEK_CUR) < 0) + { + CloseTransientFile(fd); + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not seek in foreign transaction state file: %m"))); + } + + /* write correct CRC and close file */ + if ((write(fd, &fdw_xact_crc, sizeof(pg_crc32c))) != sizeof(pg_crc32c)) + { + CloseTransientFile(fd); + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not write foreign transaction state file: %m"))); + } + + /* + * We must fsync the file because the end-of-replay checkpoint will not do + * so, there being no GXACT in shared memory yet to tell it to. + */ + if (pg_fsync(fd) != 0) + { + CloseTransientFile(fd); + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not fsync foreign transaction state file: %m"))); + } + + if (CloseTransientFile(fd) != 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not close foreign transaction file: %m"))); +} + +/* Built in functions */ +/* + * Structure to hold and iterate over the foreign transactions to be displayed + * by the built-in functions. + */ +typedef struct +{ + FDWXact fdw_xacts; + int num_xacts; + int cur_xact; +} WorkingStatus; + +/* + * pg_fdw_xact + * Produce a view with one row per prepared transaction on foreign server. + * + * This function is here so we don't have to export the + * FDWXactGlobalData struct definition. + * + */ +Datum +pg_fdw_xacts(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; + WorkingStatus *status; + char *xact_status; + + if (SRF_IS_FIRSTCALL()) + { + TupleDesc tupdesc; + MemoryContext oldcontext; + + /* create a function context for cross-call persistence */ + funcctx = SRF_FIRSTCALL_INIT(); + + /* + * Switch to memory context appropriate for multiple function calls + */ + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + /* build tupdesc for result tuples */ + /* this had better match pg_fdw_xacts view in system_views.sql */ + tupdesc = CreateTemplateTupleDesc(6, false); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "dbid", + OIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "transaction", + XIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "serverid", + OIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "userid", + OIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "status", + TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "identifier", + TEXTOID, -1, 0); + + funcctx->tuple_desc = BlessTupleDesc(tupdesc); + + /* + * Collect status information that we will format and send out as a + * result set. + */ + status = (WorkingStatus *) palloc(sizeof(WorkingStatus)); + funcctx->user_fctx = (void *) status; + + status->num_xacts = GetFDWXactList(&status->fdw_xacts); + status->cur_xact = 0; + + MemoryContextSwitchTo(oldcontext); + } + + funcctx = SRF_PERCALL_SETUP(); + status = funcctx->user_fctx; + + while (status->cur_xact < status->num_xacts) + { + FDWXact fdw_xact = &status->fdw_xacts[status->cur_xact++]; + Datum values[6]; + bool nulls[6]; + HeapTuple tuple; + Datum result; + + if (!fdw_xact->valid) + continue; + + /* + * Form tuple with appropriate data. + */ + MemSet(values, 0, sizeof(values)); + MemSet(nulls, 0, sizeof(nulls)); + + values[0] = ObjectIdGetDatum(fdw_xact->dboid); + values[1] = TransactionIdGetDatum(fdw_xact->local_xid); + values[2] = ObjectIdGetDatum(fdw_xact->serverid); + values[3] = ObjectIdGetDatum(fdw_xact->userid); + switch (fdw_xact->status) + { + case FDW_XACT_PREPARING: + xact_status = "prepared"; + break; + case FDW_XACT_COMMITTING_PREPARED: + xact_status = "committing"; + break; + case FDW_XACT_ABORTING_PREPARED: + xact_status = "aborting"; + break; + default: + xact_status = "unknown"; + break; + } + values[4] = CStringGetTextDatum(xact_status); + /* should this be really interpreted by FDW */ + values[5] = PointerGetDatum(cstring_to_text_with_len(fdw_xact->fdw_xact_id, + fdw_xact->fdw_xact_id_len)); + + tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); + result = HeapTupleGetDatum(tuple); + SRF_RETURN_NEXT(funcctx, result); + } + + SRF_RETURN_DONE(funcctx); +} + +/* + * Returns an array of all foreign prepared transactions for the user-level + * function pg_fdw_xact. + * + * The returned array and all its elements are copies of internal data + * structures, to minimize the time we need to hold the FDWXactLock. + * + * WARNING -- we return even those transactions whose information is not + * completely filled yet. The caller should filter them out if he doesn't want them. + * + * The returned array is palloc'd. + */ +static int +GetFDWXactList(FDWXact * fdw_xacts) +{ + int num_xacts; + int cnt_xacts; + + LWLockAcquire(FDWXactLock, LW_SHARED); + + if (FDWXactGlobal->numFDWXacts == 0) + { + LWLockRelease(FDWXactLock); + *fdw_xacts = NULL; + return 0; + } + + num_xacts = FDWXactGlobal->numFDWXacts; + *fdw_xacts = (FDWXact) palloc(sizeof(FDWXactData) * num_xacts); + for (cnt_xacts = 0; cnt_xacts < num_xacts; cnt_xacts++) + memcpy((*fdw_xacts) + cnt_xacts, FDWXactGlobal->fdw_xacts[cnt_xacts], + sizeof(FDWXactData)); + + LWLockRelease(FDWXactLock); + + return num_xacts; +} + +/* + * pg_fdw_xact_resolve + * a user interface to initiate foreign transaction resolution. The function + * tries to resolve the prepared transactions on foreign servers in the database + * from where it is run. + * The function prints the status of all the foreign transactions it + * encountered, whether resolved or not. + */ +Datum +pg_fdw_xact_resolve(PG_FUNCTION_ARGS) +{ + MemoryContext oldcontext; + FuncCallContext *funcctx; + WorkingStatus *status; + char *xact_status; + List *entries_to_resolve; + + if (SRF_IS_FIRSTCALL()) + { + TupleDesc tupdesc; + + /* We will be modifying the shared memory. Prepare to clean up on exit */ + if (!fdwXactExitRegistered) + { + before_shmem_exit(AtProcExit_FDWXact, 0); + fdwXactExitRegistered = true; + } + + /* Allocate space for and prepare the returning set */ + /* create a function context for cross-call persistence */ + funcctx = SRF_FIRSTCALL_INIT(); + /* Switch to memory context appropriate for multiple function calls */ + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + /* build tupdesc for result tuples */ + tupdesc = CreateTemplateTupleDesc(6, false); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "dbid", + OIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "transaction", + XIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "serverid", + OIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 4, "userid", + OIDOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 5, "status", + TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 6, "identifier", + TEXTOID, -1, 0); + + funcctx->tuple_desc = BlessTupleDesc(tupdesc); + + /* + * Collect status information that we will format and send out as a + * result set. + */ + status = (WorkingStatus *) palloc(sizeof(WorkingStatus)); + funcctx->user_fctx = (void *) status; + status->fdw_xacts = (FDWXact) palloc(sizeof(FDWXactData) * FDWXactGlobal->numFDWXacts); + status->num_xacts = 0; + status->cur_xact = 0; + + /* Done preparation for the result. */ + MemoryContextSwitchTo(oldcontext); + + /* + * Get entries whose foreign servers are part of the database where + * this function was called. We can get information about only such + * foreign servers. The function will lock the entries. The entries + * which are locked by other backends and whose foreign servers belong + * to this database are left out, since we can not work on those. + */ + search_fdw_xact(InvalidTransactionId, MyDatabaseId, InvalidOid, InvalidOid, + &entries_to_resolve); + + /* Work to resolve the resolvable entries */ + while (entries_to_resolve) + { + FDWXact fdw_xact = linitial(entries_to_resolve); + + /* Remove the entry as we will not use it again */ + entries_to_resolve = list_delete_first(entries_to_resolve); + + /* Copy the data for the sake of result. */ + memcpy(status->fdw_xacts + status->num_xacts++, + fdw_xact, sizeof(FDWXactData)); + + if (fdw_xact->status == FDW_XACT_COMMITTING_PREPARED || + fdw_xact->status == FDW_XACT_ABORTING_PREPARED) + { + /* + * We have already decided what to do with the foreign + * transaction nothing to be done. + */ + } + else if (TransactionIdDidCommit(fdw_xact->local_xid)) + fdw_xact->status = FDW_XACT_COMMITTING_PREPARED; + else if (TransactionIdDidAbort(fdw_xact->local_xid)) + fdw_xact->status = FDW_XACT_ABORTING_PREPARED; + else if (!TransactionIdIsInProgress(fdw_xact->local_xid)) + { + /* + * The transaction is in progress but not on any of the + * backends. So probably, it crashed before actual abort or + * commit. So assume it to be aborted. + */ + fdw_xact->status = FDW_XACT_ABORTING_PREPARED; + } + else + { + /* + * Local transaction is in progress, should not resolve the + * foreign transaction. This can happen when the foreign + * transaction is prepared as part of a local prepared + * transaction. Just continue with the next one. + */ + unlock_fdw_xact(fdw_xact); + continue; + } + + /* + * Resolve the foreign transaction. If resolution was not + * successful, unlock the entry so that someone else can pick it + * up + */ + if (!resolve_fdw_xact(fdw_xact, get_prepared_foreign_xact_resolver(fdw_xact))) + unlock_fdw_xact(fdw_xact); + else + /* Update the status in the result set */ + status->fdw_xacts[status->num_xacts - 1].status = FDW_XACT_RESOLVED; + } + } + + /* Print the result set */ + funcctx = SRF_PERCALL_SETUP(); + status = funcctx->user_fctx; + + while (status->cur_xact < status->num_xacts) + { + FDWXact fdw_xact = &status->fdw_xacts[status->cur_xact++]; + Datum values[6]; + bool nulls[6]; + HeapTuple tuple; + Datum result; + + if (!fdw_xact->valid) + continue; + + /* + * Form tuple with appropriate data. + */ + MemSet(values, 0, sizeof(values)); + MemSet(nulls, 0, sizeof(nulls)); + + values[0] = ObjectIdGetDatum(fdw_xact->dboid); + values[1] = TransactionIdGetDatum(fdw_xact->local_xid); + values[2] = ObjectIdGetDatum(fdw_xact->serverid); + values[3] = ObjectIdGetDatum(fdw_xact->userid); + switch (fdw_xact->status) + { + case FDW_XACT_PREPARING: + xact_status = "preparing"; + break; + case FDW_XACT_COMMITTING_PREPARED: + xact_status = "committing"; + break; + case FDW_XACT_ABORTING_PREPARED: + xact_status = "aborting"; + break; + case FDW_XACT_RESOLVED: + xact_status = "resolved"; + break; + default: + xact_status = "unknown"; + break; + } + values[4] = CStringGetTextDatum(xact_status); + /* should this be really interpreted by FDW? */ + values[5] = PointerGetDatum(cstring_to_text_with_len(fdw_xact->fdw_xact_id, + fdw_xact->fdw_xact_id_len)); + + tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); + result = HeapTupleGetDatum(tuple); + SRF_RETURN_NEXT(funcctx, result); + } + + SRF_RETURN_DONE(funcctx); +} + +/* + * Built-in function to remove prepared foreign transaction entry/s without + * resolving. The function gives a way to forget about such prepared + * transaction in case + * 1. The foreign server where it is prepared is no longer available + * 2. The user which prepared this transaction needs to be dropped + * 3. PITR is recovering before a transaction id, which created the prepared + * foreign transaction + * 4. The database containing the entries needs to be dropped + * + * Or any such conditions in which resolution is no longer possible. + * + * The function accepts 4 arguments transaction id, dbid, serverid and userid, + * which define the criteria in the same way as search_fdw_xact(). The entries + * matching the criteria are removed. The function does not remove an entry + * which is locked by some other backend. + */ +Datum +pg_fdw_xact_remove(PG_FUNCTION_ARGS) +{ +/* Some #defines only for this function to deal with the arguments */ +#define XID_ARGNUM 0 +#define DBID_ARGNUM 1 +#define SRVID_ARGNUM 2 +#define USRID_ARGNUM 3 + + TransactionId xid; + Oid dbid; + Oid serverid; + Oid userid; + List *entries_to_remove; + + xid = PG_ARGISNULL(XID_ARGNUM) ? InvalidTransactionId : + DatumGetTransactionId(PG_GETARG_DATUM(XID_ARGNUM)); + dbid = PG_ARGISNULL(DBID_ARGNUM) ? InvalidOid : + PG_GETARG_OID(DBID_ARGNUM); + serverid = PG_ARGISNULL(SRVID_ARGNUM) ? InvalidOid : + PG_GETARG_OID(SRVID_ARGNUM); + userid = PG_ARGISNULL(USRID_ARGNUM) ? InvalidOid : + PG_GETARG_OID(USRID_ARGNUM); + + search_fdw_xact(xid, dbid, serverid, userid, &entries_to_remove); + + while (entries_to_remove) + { + FDWXact fdw_xact = linitial(entries_to_remove); + + entries_to_remove = list_delete_first(entries_to_remove); + + remove_fdw_xact(fdw_xact); + } + + PG_RETURN_VOID(); +} + +/* + * Code dealing with the on disk files used to store foreign transaction + * information. + */ + +/* + * ReadFDWXactFile + * Read the foreign transction state file and return the contents in a + * structure allocated in-memory. The structure can be later freed by the + * caller. + */ +static FDWXactOnDiskData * +ReadFDWXactFile(TransactionId xid, Oid serverid, Oid userid) +{ + char path[MAXPGPATH]; + int fd; + FDWXactOnDiskData *fdw_xact_file_data; + struct stat stat; + uint32 crc_offset; + pg_crc32c calc_crc; + pg_crc32c file_crc; + char *buf; + + FDWXactFilePath(path, xid, serverid, userid); + + fd = OpenTransientFile(path, O_RDONLY | PG_BINARY, 0); + if (fd < 0) + ereport(ERROR, + (errcode_for_file_access(), + errmsg("could not open FDW transaction state file \"%s\": %m", + path))); + + /* + * Check file length. We can determine a lower bound pretty easily. We + * set an upper bound to avoid palloc() failure on a corrupt file, though + * we can't guarantee that we won't get an out of memory error anyway, + * even on a valid file. + */ + if (fstat(fd, &stat)) + { + CloseTransientFile(fd); + + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not stat FDW transaction state file \"%s\": %m", + path))); + return NULL; + } + + if (stat.st_size < offsetof(FDWXactOnDiskData, fdw_xact_id) || + stat.st_size > MaxAllocSize) + { + CloseTransientFile(fd); + ereport(WARNING, + (errcode_for_file_access(), + errmsg("Too large FDW transaction state file \"%s\": %m", + path))); + return NULL; + } + + buf = (char *) palloc(stat.st_size); + fdw_xact_file_data = (FDWXactOnDiskData *) buf; + crc_offset = stat.st_size - sizeof(pg_crc32c); + /* Slurp the file */ + if (read(fd, fdw_xact_file_data, stat.st_size) != stat.st_size) + { + CloseTransientFile(fd); + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not read FDW transaction state file \"%s\": %m", + path))); + pfree(fdw_xact_file_data); + return NULL; + } + + CloseTransientFile(fd); + + /* + * Check the CRC. + */ + INIT_CRC32C(calc_crc); + COMP_CRC32C(calc_crc, buf, crc_offset); + FIN_CRC32C(calc_crc); + + file_crc = *((pg_crc32c *) (buf + crc_offset)); + + if (!EQ_CRC32C(calc_crc, file_crc)) + { + pfree(buf); + return NULL; + } + + if (fdw_xact_file_data->serverid != serverid || + fdw_xact_file_data->userid != userid || + fdw_xact_file_data->local_xid != xid) + { + ereport(WARNING, + (errmsg("removing corrupt foreign transaction state file \"%s\"", + path))); + CloseTransientFile(fd); + pfree(buf); + return NULL; + } + + return fdw_xact_file_data; +} + +/* + * PrescanFDWXacts + * + * Read the foreign prepared transactions directory for oldest active + * transaction. The transactions corresponding to the xids in this directory + * are not necessarily active per say locally. But we still need those XIDs to + * be alive so that + * 1. we can determine whether they are committed or aborted + * 2. the file name contains xid which shouldn't get used again to avoid + * conflicting file names. + * + * The function accepts the oldest active xid determined by other functions + * (e.g. PrescanPreparedTransactions()). It then compares every xid it comes + * across while scanning foreign prepared transactions directory with the oldest + * active xid. It returns the oldest of those xids or oldest active xid + * whichever is older. + * + * If any foreign prepared transaction is part of a future transaction (PITR), + * the function removes the corresponding file as + * 1. We can not know the status of the local transaction which prepared this + * foreign transaction + * 2. The foreign server or the user may not be available as per new timeline + * + * Anyway, the local transaction which prepared the foreign prepared transaction + * does not exist as per the new timeline, so it's better to forget the foreign + * prepared transaction as well. + */ +TransactionId +PrescanFDWXacts(TransactionId oldestActiveXid) +{ + TransactionId nextXid = ShmemVariableCache->nextXid; + DIR *cldir; + struct dirent *clde; + + cldir = AllocateDir(FDW_XACTS_DIR); + while ((clde = ReadDir(cldir, FDW_XACTS_DIR)) != NULL) + { + if (strlen(clde->d_name) == FDW_XACT_FILE_NAME_LEN && + strspn(clde->d_name, "0123456789ABCDEF_") == FDW_XACT_FILE_NAME_LEN) + { + Oid serverid; + Oid userid; + TransactionId local_xid; + + sscanf(clde->d_name, "%08x_%08x_%08x", &local_xid, &serverid, + &userid); + + /* + * Remove a foreign prepared transaction file corresponding to an + * XID, which is too new. + */ + if (TransactionIdFollowsOrEquals(local_xid, nextXid)) + { + ereport(WARNING, + (errmsg("removing future foreign prepared transaction file \"%s\"", + clde->d_name))); + RemoveFDWXactFile(local_xid, serverid, userid, true); + continue; + } + + if (TransactionIdPrecedesOrEquals(local_xid, oldestActiveXid)) + oldestActiveXid = local_xid; + } + } + + FreeDir(cldir); + return oldestActiveXid; +} + +/* + * RecoverFDWXacts + * Read the foreign prepared transaction information and set it up for further + * usage. + */ +void +RecoverFDWXacts(void) +{ + DIR *cldir; + struct dirent *clde; + + cldir = AllocateDir(FDW_XACTS_DIR); + while ((clde = ReadDir(cldir, FDW_XACTS_DIR)) != NULL) + { + if (strlen(clde->d_name) == FDW_XACT_FILE_NAME_LEN && + strspn(clde->d_name, "0123456789ABCDEF_") == FDW_XACT_FILE_NAME_LEN) + { + Oid serverid; + Oid userid; + TransactionId local_xid; + FDWXactOnDiskData *fdw_xact_file_data; + FDWXact fdw_xact; + + sscanf(clde->d_name, "%08x_%08x_%08x", &local_xid, &serverid, + &userid); + + fdw_xact_file_data = ReadFDWXactFile(local_xid, serverid, userid); + + if (!fdw_xact_file_data) + { + ereport(WARNING, + (errmsg("Removing corrupt foreign transaction file \"%s\"", + clde->d_name))); + RemoveFDWXactFile(local_xid, serverid, userid, false); + continue; + } + + ereport(LOG, + (errmsg("recovering foreign transaction entry for xid %u, foreign server %u and user %u", + local_xid, serverid, userid))); + + fdw_xact = get_fdw_xact(local_xid, serverid, userid); + + LWLockAcquire(FDWXactLock, LW_EXCLUSIVE); + if (!fdw_xact) + { + /* + * Add this entry into the table of foreign transactions. The + * status of the transaction is set as preparing, since we do not + * know the exact status right now. Resolver will set it later + * based on the status of local transaction which prepared this + * foreign transaction. + */ + fdw_xact = insert_fdw_xact(fdw_xact_file_data->dboid, local_xid, + serverid, userid, + fdw_xact_file_data->umid, + fdw_xact_file_data->fdw_xact_id_len, + fdw_xact_file_data->fdw_xact_id); + fdw_xact->locking_backend = MyBackendId; + fdw_xact->status = FDW_XACT_PREPARING; + } + else + { + Assert(fdw_xact->inredo); + fdw_xact->inredo = false; + } + + /* Mark the entry as ready */ + fdw_xact->valid = true; + /* Already synced to disk */ + fdw_xact->ondisk = true; + pfree(fdw_xact_file_data); + LWLockRelease(FDWXactLock); + } + } + + FreeDir(cldir); +} + +/* + * Remove the foreign transaction file for given entry. + * + * If giveWarning is false, do not complain about file-not-present; + * this is an expected case during WAL replay. + */ +static void +RemoveFDWXactFile(TransactionId xid, Oid serverid, Oid userid, bool giveWarning) +{ + char path[MAXPGPATH]; + + FDWXactFilePath(path, xid, serverid, userid); + if (unlink(path)) + if (errno != ENOENT || giveWarning) + ereport(WARNING, + (errcode_for_file_access(), + errmsg("could not remove foreign transaction state file \"%s\": %m", + path))); +} + +/* + * FDWXactRedoAdd + * + * Store pointer to the start/end of the WAL record along with the xid in + * a fdw_xact entry in shared memory FDWXactData structure. + */ +void +FDWXactRedoAdd(XLogReaderState *record) +{ + FDWXactOnDiskData *fdw_xact_data = (FDWXactOnDiskData *) XLogRecGetData(record); + FDWXact fdw_xact; + + Assert(RecoveryInProgress()); + + LWLockAcquire(FDWXactLock, LW_EXCLUSIVE); + fdw_xact = insert_fdw_xact(fdw_xact_data->dboid, fdw_xact_data->local_xid, + fdw_xact_data->serverid, fdw_xact_data->userid, + fdw_xact_data->umid, fdw_xact_data->fdw_xact_id_len, + fdw_xact_data->fdw_xact_id); + fdw_xact->status = FDW_XACT_PREPARING; + fdw_xact->fdw_xact_start_lsn = record->ReadRecPtr; + fdw_xact->fdw_xact_end_lsn = record->EndRecPtr; + fdw_xact->inredo = true; + LWLockRelease(FDWXactLock); +} +/* + * FDWXactRedoRemove + * + * Remove the corresponding fdw_xact entry from FDWXactGlobal. + * Also remove fdw_xact file if a foreign transaction was saved + * via an earlier checkpoint. + */ +void +FDWXactRedoRemove(TransactionId xid, Oid serverid, Oid userid) +{ + FDWXact fdw_xact; + + Assert(RecoveryInProgress()); + + fdw_xact = get_fdw_xact(xid, serverid, userid); + + if (fdw_xact) + { + /* Now we can clean up any files we already left */ + Assert(fdw_xact->inredo); + remove_fdw_xact(fdw_xact); + } + else + { + /* + * Entry could be on disk. Call with giveWarning = false + * since it can be expected during replay. + */ + RemoveFDWXactFile(xid, serverid, userid, false); + } +} diff --git a/src/backend/access/transam/rmgr.c b/src/backend/access/transam/rmgr.c index 9368b56..c10a027 100644 --- a/src/backend/access/transam/rmgr.c +++ b/src/backend/access/transam/rmgr.c @@ -9,6 +9,7 @@ #include "access/clog.h" #include "access/commit_ts.h" +#include "access/fdw_xact.h" #include "access/ginxlog.h" #include "access/gistxlog.h" #include "access/generic_xlog.h" diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c index cfaf8da..ff268bc 100644 --- a/src/backend/access/transam/twophase.c +++ b/src/backend/access/transam/twophase.c @@ -77,6 +77,7 @@ #include #include "access/commit_ts.h" +#include "access/fdw_xact.h" #include "access/htup_details.h" #include "access/subtrans.h" #include "access/transam.h" @@ -1498,6 +1499,12 @@ FinishPreparedTransaction(const char *gid, bool isCommit) PredicateLockTwoPhaseFinish(xid, isCommit); + /* + * Commit/Rollback the foreign transactions prepared as part of this + * prepared transaction. + */ + FDWXactTwoPhaseFinish(isCommit, xid); + /* Count the prepared xact as committed or aborted */ AtEOXact_PgStat(isCommit); diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c index 52408fc..563333d 100644 --- a/src/backend/access/transam/xact.c +++ b/src/backend/access/transam/xact.c @@ -21,6 +21,7 @@ #include #include "access/commit_ts.h" +#include "access/fdw_xact.h" #include "access/multixact.h" #include "access/parallel.h" #include "access/subtrans.h" @@ -118,6 +119,9 @@ TransactionId *ParallelCurrentXids; */ int MyXactFlags; +/* Transaction do the write on local node */ +bool XactWriteLocalNode = false; + /* * transaction states - transaction state from server perspective */ @@ -1983,6 +1987,9 @@ CommitTransaction(void) break; } + /* Pre-commit step for foreign transcations */ + PreCommit_FDWXacts(); + CallXactCallbacks(is_parallel_worker ? XACT_EVENT_PARALLEL_PRE_COMMIT : XACT_EVENT_PRE_COMMIT); @@ -2139,6 +2146,7 @@ CommitTransaction(void) AtEOXact_PgStat(true); AtEOXact_Snapshot(true, false); AtEOXact_ApplyLauncher(true); + AtEOXact_FDWXacts(true); pgstat_report_xact_timestamp(0); CurrentResourceOwner = NULL; @@ -2160,6 +2168,8 @@ CommitTransaction(void) XactTopTransactionId = InvalidTransactionId; nParallelCurrentXids = 0; + UnregisterTransactionLocalNode(); + /* * done with commit processing, set current transaction state back to * default @@ -2226,6 +2236,9 @@ PrepareTransaction(void) * the transaction-abort path. */ + /* Prepare step for foreign transactions */ + AtPrepare_FDWXacts(); + /* Shut down the deferred-trigger manager */ AfterTriggerEndXact(true); @@ -2438,6 +2451,8 @@ PrepareTransaction(void) XactTopTransactionId = InvalidTransactionId; nParallelCurrentXids = 0; + UnregisterTransactionLocalNode(); + /* * done with 1st phase commit processing, set current transaction state * back to default @@ -2620,9 +2635,12 @@ AbortTransaction(void) AtEOXact_HashTables(false); AtEOXact_PgStat(false); AtEOXact_ApplyLauncher(false); + AtEOXact_FDWXacts(false); pgstat_report_xact_timestamp(0); } + UnregisterTransactionLocalNode(); + /* * State remains TRANS_ABORT until CleanupTransaction(). */ @@ -4453,6 +4471,32 @@ AbortOutOfAnyTransaction(void) } /* + * RegisterTransactionLocalNode --- remember to wrote on local node + */ +void +RegisterTransactionLocalNode(void) +{ + /* Quick exits if no need to remember */ + if (max_prepared_foreign_xacts == 0) + return; + + XactWriteLocalNode = true; +} + +/* + * UnregisterTransactionLocalNode --- forget to wrote on local node + */ +void +UnregisterTransactionLocalNode(void) +{ + /* Quick exits if no need to remember */ + if (max_prepared_foreign_xacts == 0) + return; + + XactWriteLocalNode = false; +} + +/* * IsTransactionBlock --- are we within a transaction block? */ bool diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index dd028a1..581fae3 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -24,6 +24,7 @@ #include "access/clog.h" #include "access/commit_ts.h" +#include "access/fdw_xact.h" #include "access/multixact.h" #include "access/rewriteheap.h" #include "access/subtrans.h" @@ -5174,6 +5175,7 @@ BootStrapXLOG(void) ControlFile->MaxConnections = MaxConnections; ControlFile->max_worker_processes = max_worker_processes; ControlFile->max_prepared_xacts = max_prepared_xacts; + ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts; ControlFile->max_locks_per_xact = max_locks_per_xact; ControlFile->wal_level = wal_level; ControlFile->wal_log_hints = wal_log_hints; @@ -6264,6 +6266,9 @@ CheckRequiredParameterValues(void) RecoveryRequiresIntParameter("max_locks_per_transaction", max_locks_per_xact, ControlFile->max_locks_per_xact); + RecoveryRequiresIntParameter("max_prepared_foreign_transactions", + max_prepared_foreign_xacts, + ControlFile->max_prepared_foreign_xacts); } } @@ -6963,7 +6968,10 @@ StartupXLOG(void) InitRecoveryTransactionEnvironment(); if (wasShutdown) + { oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids); + oldestActiveXID = PrescanFDWXacts(oldestActiveXID); + } else oldestActiveXID = checkPoint.oldestActiveXid; Assert(TransactionIdIsValid(oldestActiveXID)); @@ -7588,6 +7596,7 @@ StartupXLOG(void) /* Pre-scan prepared transactions to find out the range of XIDs present */ oldestActiveXID = PrescanPreparedTransactions(NULL, NULL); + oldestActiveXID = PrescanFDWXacts(oldestActiveXID); /* * Update full_page_writes in shared memory and write an XLOG_FPW_CHANGE @@ -7775,6 +7784,9 @@ StartupXLOG(void) /* Reload shared-memory state for prepared transactions */ RecoverPreparedTransactions(); + /* Recover foreign transaction state and insert into shared-memory. */ + RecoverFDWXacts(); + /* * Shutdown the recovery environment. This must occur after * RecoverPreparedTransactions(), see notes for lock_twophase_recover() @@ -9101,6 +9113,11 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags) CheckPointReplicationOrigin(); /* We deliberately delay 2PC checkpointing as long as possible */ CheckPointTwoPhase(checkPointRedo); + /* + * We deliberately delay foreign transaction checkpointing as long as + * possible. + */ + CheckPointFDWXact(checkPointRedo); } /* @@ -9538,7 +9555,8 @@ XLogReportParameters(void) max_worker_processes != ControlFile->max_worker_processes || max_prepared_xacts != ControlFile->max_prepared_xacts || max_locks_per_xact != ControlFile->max_locks_per_xact || - track_commit_timestamp != ControlFile->track_commit_timestamp) + track_commit_timestamp != ControlFile->track_commit_timestamp || + max_prepared_foreign_xacts != ControlFile->max_prepared_foreign_xacts) { /* * The change in number of backend slots doesn't need to be WAL-logged @@ -9555,6 +9573,7 @@ XLogReportParameters(void) xlrec.MaxConnections = MaxConnections; xlrec.max_worker_processes = max_worker_processes; xlrec.max_prepared_xacts = max_prepared_xacts; + xlrec.max_prepared_foreign_xacts = max_prepared_foreign_xacts; xlrec.max_locks_per_xact = max_locks_per_xact; xlrec.wal_level = wal_level; xlrec.wal_log_hints = wal_log_hints; @@ -9570,6 +9589,7 @@ XLogReportParameters(void) ControlFile->MaxConnections = MaxConnections; ControlFile->max_worker_processes = max_worker_processes; ControlFile->max_prepared_xacts = max_prepared_xacts; + ControlFile->max_prepared_foreign_xacts = max_prepared_foreign_xacts; ControlFile->max_locks_per_xact = max_locks_per_xact; ControlFile->wal_level = wal_level; ControlFile->wal_log_hints = wal_log_hints; @@ -9767,6 +9787,7 @@ xlog_redo(XLogReaderState *record) RunningTransactionsData running; oldestActiveXID = PrescanPreparedTransactions(&xids, &nxids); + oldestActiveXID = PrescanFDWXacts(oldestActiveXID); /* * Construct a RunningTransactions snapshot representing a shut @@ -9956,6 +9977,7 @@ xlog_redo(XLogReaderState *record) ControlFile->MaxConnections = xlrec.MaxConnections; ControlFile->max_worker_processes = xlrec.max_worker_processes; ControlFile->max_prepared_xacts = xlrec.max_prepared_xacts; + ControlFile->max_prepared_foreign_xacts = xlrec.max_prepared_foreign_xacts; ControlFile->max_locks_per_xact = xlrec.max_locks_per_xact; ControlFile->wal_level = xlrec.wal_level; ControlFile->wal_log_hints = xlrec.wal_log_hints; diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index 8287de9..8dbc347 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -17,6 +17,7 @@ #include #include +#include "access/fdw_xact.h" #include "access/htup_details.h" #include "access/xact.h" #include "access/xlog_internal.h" diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index dc40cde..44c996c 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -291,6 +291,9 @@ CREATE VIEW pg_prepared_xacts AS CREATE VIEW pg_prepared_statements AS SELECT * FROM pg_prepared_statement() AS P; +CREATE VIEW pg_fdw_xacts AS + SELECT * FROM pg_fdw_xacts() AS F; + CREATE VIEW pg_seclabels AS SELECT l.objoid, l.classoid, l.objsubid, diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index 9ad9915..8d6e240 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -13,6 +13,7 @@ */ #include "postgres.h" +#include "access/fdw_xact.h" #include "access/heapam.h" #include "access/htup_details.h" #include "access/reloptions.h" @@ -1093,6 +1094,20 @@ RemoveForeignServerById(Oid srvId) if (!HeapTupleIsValid(tp)) elog(ERROR, "cache lookup failed for foreign server %u", srvId); + /* + * Check if the foreign server has any foreign transaction prepared on it. + * If there is one, and it gets dropped, we will not have any chance to + * resolve that transaction. + */ + if (fdw_xact_exists(InvalidTransactionId, MyDatabaseId, srvId, InvalidOid)) + { + Form_pg_foreign_server srvForm; + srvForm = (Form_pg_foreign_server) GETSTRUCT(tp); + ereport(ERROR, + (errmsg("server \"%s\" has unresolved prepared transactions on it", + NameStr(srvForm->srvname)))); + } + CatalogTupleDelete(rel, &tp->t_self); ReleaseSysCache(tp); @@ -1403,6 +1418,17 @@ RemoveUserMapping(DropUserMappingStmt *stmt) user_mapping_ddl_aclcheck(useId, srv->serverid, srv->servername); /* + * If there is a foreign prepared transaction with this user mapping, + * dropping the user mapping might result in dangling prepared + * transaction. + */ + if (fdw_xact_exists(InvalidTransactionId, MyDatabaseId, srv->serverid, + useId)) + ereport(ERROR, + (errmsg("server \"%s\" has unresolved prepared transaction for user \"%s\"", + srv->servername, MappingUserName(useId)))); + + /* * Do the deletion */ object.classId = UserMappingRelationId; diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index 845c409..3a37aa9 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -487,6 +487,9 @@ ExecInsert(ModifyTableState *mtstate, if (resultRelationDesc->rd_att->constr || check_partition_constr) ExecConstraints(resultRelInfo, slot, estate); + /* Remember to wrote on local node for foreign transaction */ + RegisterTransactionLocalNode(); + if (onconflict != ONCONFLICT_NONE && resultRelInfo->ri_NumIndices > 0) { /* Perform a speculative insertion. */ @@ -750,6 +753,9 @@ ExecDelete(ModifyTableState *mtstate, } else { + /* Remember to wrote on local node for foreign transaction */ + RegisterTransactionLocalNode(); + /* * delete the tuple * @@ -1049,6 +1055,9 @@ lreplace:; if (resultRelationDesc->rd_att->constr || resultRelInfo->ri_PartitionCheck) ExecConstraints(resultRelInfo, slot, estate); + /* Remember to wrote on local node for foreign transaction */ + RegisterTransactionLocalNode(); + /* * replace the heap tuple * diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c index 486fd0c..a773b38 100644 --- a/src/backend/replication/logical/decode.c +++ b/src/backend/replication/logical/decode.c @@ -148,6 +148,7 @@ LogicalDecodingProcessRecord(LogicalDecodingContext *ctx, XLogReaderState *recor case RM_SPGIST_ID: case RM_BRIN_ID: case RM_COMMIT_TS_ID: + case RM_FDW_XACT_ID: case RM_REPLORIGIN_ID: case RM_GENERIC_ID: /* just deal with xid, and done */ diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index 2d1ed14..f32db3a 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -16,6 +16,7 @@ #include "access/clog.h" #include "access/commit_ts.h" +#include "access/fdw_xact.h" #include "access/heapam.h" #include "access/multixact.h" #include "access/nbtree.h" @@ -150,6 +151,7 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port) size = add_size(size, SyncScanShmemSize()); size = add_size(size, AsyncShmemSize()); size = add_size(size, BackendRandomShmemSize()); + size = add_size(size, FDWXactShmemSize()); #ifdef EXEC_BACKEND size = add_size(size, ShmemBackendArraySize()); #endif @@ -270,6 +272,7 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port) SyncScanShmemInit(); AsyncShmemInit(); BackendRandomShmemInit(); + FDWXactShmemInit(); #ifdef EXEC_BACKEND diff --git a/src/backend/storage/lmgr/lwlocknames.txt b/src/backend/storage/lmgr/lwlocknames.txt index e6025ec..8e7028a 100644 --- a/src/backend/storage/lmgr/lwlocknames.txt +++ b/src/backend/storage/lmgr/lwlocknames.txt @@ -50,3 +50,4 @@ OldSnapshotTimeMapLock 42 BackendRandomLock 43 LogicalRepWorkerLock 44 CLogTruncationLock 45 +FDWXactLock 46 diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 47a5f25..1f90cda 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -27,6 +27,7 @@ #endif #include "access/commit_ts.h" +#include "access/fdw_xact.h" #include "access/gin.h" #include "access/rmgr.h" #include "access/transam.h" @@ -2065,6 +2066,19 @@ static struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + /* + * See also CheckRequiredParameterValues() if this parameter changes + */ + { + {"max_prepared_foreign_transactions", PGC_POSTMASTER, RESOURCES_MEM, + gettext_noop("Sets the maximum number of simultaneously prepared transactions on foreign servers."), + NULL + }, + &max_prepared_foreign_xacts, + 0, 0, INT_MAX, + NULL, NULL, NULL + }, + #ifdef LOCK_DEBUG { {"trace_lock_oidmin", PGC_SUSET, DEVELOPER_OPTIONS, diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 8ba6b1d..2096cd3 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -119,6 +119,12 @@ # (change requires restart) # Caution: it is not advisable to set max_prepared_transactions nonzero unless # you actively intend to use prepared transactions. +#max_prepared_foreign_transactions = 0 # zero disables the feature + # (change requires restart) +# Note: Increasing max_prepared_foreign_transactions costs ~600(?) bytes of shared memory +# per foreign transaction slot. +# It is not advisable to set max_prepared_foreign_transactions nonzero unless you +# actively intend to use atomic foreign transactions feature. #work_mem = 4MB # min 64kB #maintenance_work_mem = 64MB # min 1MB #replacement_sort_tuples = 150000 # limits use of replacement selection sort diff --git a/src/backend/utils/probes.d b/src/backend/utils/probes.d index 214dc71..af2c627 100644 --- a/src/backend/utils/probes.d +++ b/src/backend/utils/probes.d @@ -81,6 +81,8 @@ provider postgresql { probe multixact__checkpoint__done(bool); probe twophase__checkpoint__start(); probe twophase__checkpoint__done(); + probe fdwxact__checkpoint__start(); + probe fdwxact__checkpoint__done(); probe smgr__md__read__start(ForkNumber, BlockNumber, Oid, Oid, Oid, int); probe smgr__md__read__done(ForkNumber, BlockNumber, Oid, Oid, Oid, int, int, int); diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 27fcf5a..9404506 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -208,6 +208,7 @@ static const char *const subdirs[] = { "pg_snapshots", "pg_subtrans", "pg_twophase", + "pg_fdw_xact", "pg_multixact", "pg_multixact/members", "pg_multixact/offsets", diff --git a/src/bin/pg_controldata/pg_controldata.c b/src/bin/pg_controldata/pg_controldata.c index 8cc4fb0..d8a7065 100644 --- a/src/bin/pg_controldata/pg_controldata.c +++ b/src/bin/pg_controldata/pg_controldata.c @@ -287,6 +287,8 @@ main(int argc, char *argv[]) ControlFile->max_worker_processes); printf(_("max_prepared_xacts setting: %d\n"), ControlFile->max_prepared_xacts); + printf(_("max_prepared_foreign_xacts setting: %d\n"), + ControlFile->max_prepared_foreign_xacts); printf(_("max_locks_per_xact setting: %d\n"), ControlFile->max_locks_per_xact); printf(_("track_commit_timestamp setting: %s\n"), diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c index 25d5547..168cce8 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c @@ -672,6 +672,7 @@ GuessControlValues(void) ControlFile.MaxConnections = 100; ControlFile.max_worker_processes = 8; ControlFile.max_prepared_xacts = 0; + ControlFile.max_prepared_foreign_xacts = 0; ControlFile.max_locks_per_xact = 64; ControlFile.maxAlign = MAXIMUM_ALIGNOF; @@ -894,6 +895,7 @@ RewriteControlFile(void) ControlFile.MaxConnections = 100; ControlFile.max_worker_processes = 8; ControlFile.max_prepared_xacts = 0; + ControlFile.max_prepared_foreign_xacts = 0; ControlFile.max_locks_per_xact = 64; /* Now we can force the recorded xlog seg size to the right thing. */ diff --git a/src/bin/pg_waldump/rmgrdesc.c b/src/bin/pg_waldump/rmgrdesc.c index 852d8ca..41eed51 100644 --- a/src/bin/pg_waldump/rmgrdesc.c +++ b/src/bin/pg_waldump/rmgrdesc.c @@ -8,6 +8,7 @@ #define FRONTEND 1 #include "postgres.h" +#include "access/fdw_xact.h" #include "access/brin_xlog.h" #include "access/clog.h" #include "access/commit_ts.h" diff --git a/src/include/access/fdw_xact.h b/src/include/access/fdw_xact.h new file mode 100644 index 0000000..69b74af --- /dev/null +++ b/src/include/access/fdw_xact.h @@ -0,0 +1,76 @@ +/* + * fdw_xact.h + * + * PostgreSQL distributed transaction manager + * + * Portions Copyright (c) 2017, PostgreSQL Global Development Group + * + * src/include/access/fdw_xact.h + */ +#ifndef FDW_XACT_H +#define FDW_XACT_H + +#include "storage/backendid.h" +#include "foreign/foreign.h" +#include "access/xlogreader.h" +#include "lib/stringinfo.h" +#include "nodes/pg_list.h" + +/* + * On disk file structure + */ +typedef struct +{ + Oid dboid; /* database oid where to find foreign server + * and user mapping */ + TransactionId local_xid; + Oid serverid; /* foreign server where transaction takes + * place */ + Oid userid; /* user who initiated the foreign transaction */ + Oid umid; + uint32 fdw_xact_id_len;/* Length of the value stored in the next + * field */ + /* This should always be the last member */ + char fdw_xact_id[FLEXIBLE_ARRAY_MEMBER]; /* variable length array + * to store foreign + * transaction + * information. */ +} FDWXactOnDiskData; + +typedef struct +{ + TransactionId xid; + Oid serverid; + Oid userid; + Oid dbid; +} FdwRemoveXlogRec; + +extern int max_prepared_foreign_xacts; + +/* Info types for logs related to FDW transactions */ +#define XLOG_FDW_XACT_INSERT 0x00 +#define XLOG_FDW_XACT_REMOVE 0x10 + +extern Size FDWXactShmemSize(void); +extern void FDWXactShmemInit(void); +extern void RecoverFDWXacts(void); +extern TransactionId PrescanFDWXacts(TransactionId oldestActiveXid); +extern bool fdw_xact_has_usermapping(Oid serverid, Oid userid); +extern bool fdw_xact_has_server(Oid serverid); +extern void fdw_xact_redo(XLogReaderState *record); +extern void fdw_xact_desc(StringInfo buf, XLogReaderState *record); +extern const char *fdw_xact_identify(uint8 info); +extern void AtEOXact_FDWXacts(bool is_commit); +extern void AtPrepare_FDWXacts(void); +extern void FDWXactTwoPhaseFinish(bool isCommit, TransactionId xid); +extern bool fdw_xact_exists(TransactionId xid, Oid dboid, Oid serverid, + Oid userid); +extern void CheckPointFDWXact(XLogRecPtr redo_horizon); +extern void RegisterXactForeignServer(Oid serverid, Oid userid, bool can_prepare); +extern bool FdwTwoPhaseNeeded(void); +extern void PreCommit_FDWXacts(void); +extern void FDWXactRedoAdd(XLogReaderState *record); +extern void FDWXactRedoRemove(TransactionId xid, Oid serverid, Oid userid); +extern void KnownFDWXactRecreateFiles(XLogRecPtr redo_horizon); + +#endif /* FDW_XACT_H */ diff --git a/src/include/access/rmgrlist.h b/src/include/access/rmgrlist.h index 2f43c19..62702de 100644 --- a/src/include/access/rmgrlist.h +++ b/src/include/access/rmgrlist.h @@ -47,3 +47,4 @@ PG_RMGR(RM_COMMIT_TS_ID, "CommitTs", commit_ts_redo, commit_ts_desc, commit_ts_i PG_RMGR(RM_REPLORIGIN_ID, "ReplicationOrigin", replorigin_redo, replorigin_desc, replorigin_identify, NULL, NULL, NULL) PG_RMGR(RM_GENERIC_ID, "Generic", generic_redo, generic_desc, generic_identify, NULL, NULL, generic_mask) PG_RMGR(RM_LOGICALMSG_ID, "LogicalMessage", logicalmsg_redo, logicalmsg_desc, logicalmsg_identify, NULL, NULL, NULL) +PG_RMGR(RM_FDW_XACT_ID, "Foreign Transactions", fdw_xact_redo, fdw_xact_desc, fdw_xact_identify, NULL, NULL, NULL) diff --git a/src/include/access/xact.h b/src/include/access/xact.h index f2c10f9..b8c61a8 100644 --- a/src/include/access/xact.h +++ b/src/include/access/xact.h @@ -92,6 +92,9 @@ extern int MyXactFlags; #define XACT_FLAGS_ACQUIREDACCESSEXCLUSIVELOCK (1U << 1) +/* Foreign transaction support */ +extern bool XactWriteLocalNode; + /* * start- and end-of-transaction callbacks for dynamically loaded modules */ @@ -377,6 +380,8 @@ extern void RegisterXactCallback(XactCallback callback, void *arg); extern void UnregisterXactCallback(XactCallback callback, void *arg); extern void RegisterSubXactCallback(SubXactCallback callback, void *arg); extern void UnregisterSubXactCallback(SubXactCallback callback, void *arg); +extern void RegisterTransactionLocalNode(void); +extern void UnregisterTransactionLocalNode(void); extern int xactGetCommittedChildren(TransactionId **ptr); diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index 22a8e63..54114ae 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -227,6 +227,7 @@ typedef struct xl_parameter_change int MaxConnections; int max_worker_processes; int max_prepared_xacts; + int max_prepared_foreign_xacts; int max_locks_per_xact; int wal_level; bool wal_log_hints; diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h index 3fed3b6..3189eda 100644 --- a/src/include/catalog/pg_control.h +++ b/src/include/catalog/pg_control.h @@ -179,6 +179,7 @@ typedef struct ControlFileData int MaxConnections; int max_worker_processes; int max_prepared_xacts; + int max_prepared_foreign_xacts; int max_locks_per_xact; bool track_commit_timestamp; diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 93c031a..0f370ef 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -5420,6 +5420,12 @@ DATA(insert OID = 3992 ( dense_rank PGNSP PGUID 12 1 0 2276 0 t f f f f f i s DESCR("rank of hypothetical row without gaps"); DATA(insert OID = 3993 ( dense_rank_final PGNSP PGUID 12 1 0 2276 0 f f f f f f i s 2 0 20 "2281 2276" "{2281,2276}" "{i,v}" _null_ _null_ _null_ hypothetical_dense_rank_final _null_ _null_ _null_ )); DESCR("aggregate final function"); +DATA(insert OID = 4130 ( pg_fdw_xacts PGNSP PGUID 12 1 1000 0 0 f f f f t t v u 0 0 2249 "" "{26,28,26,26,25,25}" "{o,o,o,o,o,o}" "{dbid, transaction,serverid,userid,status,identifier}" _null_ _null_ pg_fdw_xacts _null_ _null_ _null_ )); +DESCR("view foreign transactions"); +DATA(insert OID = 4131 ( pg_fdw_xact_resolve PGNSP PGUID 12 1 1000 0 0 f f f f t t v u 0 0 2249 "" "{26, 28,26,26,25,25}" "{o,o,o,o,o,o}" "{dbid, transaction,serverid,userid,status,identifier}" _null_ _null_ pg_fdw_xact_resolve _null_ _null_ _null_ )); +DESCR("resolve foreign prepared transactions"); +DATA(insert OID = 4132 ( pg_fdw_xact_remove PGNSP PGUID 12 1 0 0 0 f f f f f f v u 4 0 2278 "28 26 26 26" _null_ _null_ "{transaction,dbid,serverid,userid}" _null_ _null_ pg_fdw_xact_remove _null_ _null_ _null_ )); +DESCR("remove foreign transactions"); /* pg_upgrade support */ DATA(insert OID = 3582 ( binary_upgrade_set_next_pg_type_oid PGNSP PGUID 12 1 0 0 0 f f f f t f v r 1 0 2278 "26" _null_ _null_ _null_ _null_ _null_ binary_upgrade_set_next_pg_type_oid _null_ _null_ _null_ )); diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h index ef0fbe6..738515d 100644 --- a/src/include/foreign/fdwapi.h +++ b/src/include/foreign/fdwapi.h @@ -12,6 +12,7 @@ #ifndef FDWAPI_H #define FDWAPI_H +#include "access/fdw_xact.h" #include "access/parallel.h" #include "nodes/execnodes.h" #include "nodes/relation.h" @@ -143,6 +144,24 @@ typedef bool (*AnalyzeForeignTable_function) (Relation relation, typedef List *(*ImportForeignSchema_function) (ImportForeignSchemaStmt *stmt, Oid serverOid); +typedef bool (*EndForeignTransaction_function) (Oid serverid, Oid userid, + Oid umid, bool is_commit); + +typedef bool (*PrepareForeignTransaction_function) (Oid serverid, Oid userid, + Oid umid, int prep_info_len, + char *prep_info); + +typedef bool (*ResolvePreparedForeignTransaction_function) (Oid serverid, + Oid userid, + Oid umid, + bool is_commit, + int prep_info_len, + char *prep_info); + +typedef char *(*GetPrepareId_function) (Oid serverid, Oid userid, + int *prep_info_len); + + typedef Size (*EstimateDSMForeignScan_function) (ForeignScanState *node, ParallelContext *pcxt); typedef void (*InitializeDSMForeignScan_function) (ForeignScanState *node, @@ -223,6 +242,12 @@ typedef struct FdwRoutine /* Support functions for IMPORT FOREIGN SCHEMA */ ImportForeignSchema_function ImportForeignSchema; + /* Support functions for foreign transactions */ + GetPrepareId_function GetPrepareId; + EndForeignTransaction_function EndForeignTransaction; + PrepareForeignTransaction_function PrepareForeignTransaction; + ResolvePreparedForeignTransaction_function ResolvePreparedForeignTransaction; + /* Support functions for parallelism under Gather node */ IsForeignScanParallelSafe_function IsForeignScanParallelSafe; EstimateDSMForeignScan_function EstimateDSMForeignScan; diff --git a/src/include/storage/proc.h b/src/include/storage/proc.h index 205f484..53852de 100644 --- a/src/include/storage/proc.h +++ b/src/include/storage/proc.h @@ -280,11 +280,12 @@ extern PGPROC *PreparedXactProcs; * We set aside some extra PGPROC structures for auxiliary processes, * ie things that aren't full-fledged backends but need shmem access. * - * Background writer, checkpointer and WAL writer run during normal operation. + * Background writer, checkpointer, WAL writer and foreign transction resolver + * run during normal operation. * Startup process and WAL receiver also consume 2 slots, but WAL writer is * launched only after startup has exited, so we only need 4 slots. */ -#define NUM_AUXILIARY_PROCS 4 +#define NUM_AUXILIARY_PROCS 5 /* configurable options */ extern int DeadlockTimeout; diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h index 762532f..a563e10 100644 --- a/src/include/utils/builtins.h +++ b/src/include/utils/builtins.h @@ -121,4 +121,9 @@ extern int32 type_maximum_size(Oid type_oid, int32 typemod); /* quote.c */ extern char *quote_literal_cstr(const char *rawstr); +/* access/transam/fdw_xact.c */ +extern Datum pg_fdw_xacts(PG_FUNCTION_ARGS); +extern Datum pg_fdw_xact_resolve(PG_FUNCTION_ARGS); +extern Datum pg_fdw_xact_remove(PG_FUNCTION_ARGS); + #endif /* BUILTINS_H */ diff --git a/src/test/recovery/Makefile b/src/test/recovery/Makefile index 142a1b8..1b28f3c 100644 --- a/src/test/recovery/Makefile +++ b/src/test/recovery/Makefile @@ -21,4 +21,4 @@ check: clean distclean maintainer-clean: rm -rf tmp_check -EXTRA_INSTALL = contrib/test_decoding +EXTRA_INSTALL = contrib/test_decoding contrib/postgres_fdw diff --git a/src/test/recovery/t/010_fdw_xact.pl b/src/test/recovery/t/010_fdw_xact.pl new file mode 100644 index 0000000..58bcefd --- /dev/null +++ b/src/test/recovery/t/010_fdw_xact.pl @@ -0,0 +1,186 @@ +# Tests for transaction involving foreign servers +use strict; +use warnings; +use PostgresNode; +use TestLib; +use Test::More tests => 9; + +# Setup master node +my $node_master = get_new_node("master"); +my $node_standby = get_new_node("standby"); + +$node_master->init(allows_streaming => 1); +$node_master->append_conf('postgresql.conf', qq( +max_prepared_foreign_transactions = 10 +max_prepared_transactions = 10 +)); +$node_master->start; + +# Take backup from master node +my $backup_name = 'master_backup'; +$node_master->backup($backup_name); + +# Set up standby node +$node_standby->init_from_backup($node_master, $backup_name, + has_streaming => 1); +$node_standby->start; + +# Set up foreign nodes +my $node_fs1 = get_new_node("fs1"); +my $node_fs2 = get_new_node("fs2"); +my $fs1_port = $node_fs1->port; +my $fs2_port = $node_fs2->port; +$node_fs1->init; +$node_fs2->init; +$node_fs1->append_conf('postgresql.conf', "max_prepared_transactions = 10"); +$node_fs2->append_conf('postgresql.conf', "max_prepared_transactions = 10"); +$node_fs1->start; +$node_fs2->start; + +# Create foreign server +$node_master->safe_psql('postgres', "CREATE EXTENSION postgres_fdw"); +$node_master->safe_psql('postgres', " +CREATE SERVER fs1 FOREIGN DATA WRAPPER postgres_fdw +OPTIONS (dbname 'postgres', port '$fs1_port', two_phase_commit 'on'); +"); +$node_master->safe_psql('postgres', " +CREATE SERVER fs2 FOREIGN DATA WRAPPER postgres_fdw +OPTIONS (dbname 'postgres', port '$fs2_port', two_phase_commit 'on'); +"); + +# Create user mapping +$node_master->safe_psql('postgres', " +CREATE USER MAPPING FOR CURRENT_USER SERVER fs1; +CREATE USER MAPPING FOR CURRENT_USER SERVER fs2; +"); + +# Ceate table on foreign server and import them. +$node_fs1->safe_psql('postgres', " +CREATE SCHEMA fs; +CREATE TABLE fs.t1 AS SELECT generate_series(1,10) AS c; +"); +$node_fs2->safe_psql('postgres', " +CREATE SCHEMA fs; +CREATE TABLE fs.t2 AS SELECT generate_series(1,10) AS c; +"); +$node_master->safe_psql('postgres', " +IMPORT FOREIGN SCHEMA fs FROM SERVER fs1 INTO public; +IMPORT FOREIGN SCHEMA fs FROM SERVER fs2 INTO public; +CREATE TABLE local_table (c int); +INSERT INTO local_table SELECT generate_series(1,10); +"); + +# Switch to synchronous replication +$node_master->psql('postgres', "ALTER SYSTEM SET synchronous_standby_names = '*'"); +$node_master->reload; + +my $result; + +# +# Prepare two transaction involving foreign servers. +# Check if we can commit and rollback transaction involving foreign servers after recovery. +# +$node_master->safe_psql('postgres', " +BEGIN; +UPDATE t1 SET c = 1 WHERE c = 1; +UPDATE t2 SET c = 2 WHERE c = 2; +PREPARE TRANSACTION 'gxid1'; +BEGIN; +UPDATE t1 SET c = 3 WHERE c = 3; +UPDATE t2 SET c = 4 WHERE c = 4; +PREPARE TRANSACTION 'gxid2'; +"); + +$node_master->stop; +$node_master->start; + +$result = $node_master->psql('postgres', "COMMIT PREPARED 'gxid1'"); +is($result, 0, 'Commit foreigin transaction after recovery'); +$result = $node_master->psql('postgres', "ROLLBACK PREPARED 'gxid2'"); +is($result, 0, 'Rollback foreigin transaction after recovery'); + +# +# Prepare two transaction involving foreign servers and shutdown master node immediately. +# Check if we can commit and rollback transaction involving foreign servers after crash recovery. +# +$node_master->safe_psql('postgres', " +BEGIN; +UPDATE t1 SET c = 5 WHERE c = 5; +UPDATE t2 SET c = 6 WHERE c = 6; +PREPARE TRANSACTION 'gxid1'; +BEGIN; +UPDATE t1 SET c = 7 WHERE c = 7; +UPDATE t2 SET c = 8 WHERE c = 8; +PREPARE TRANSACTION 'gxid2'; +"); + +$node_master->teardown_node; +$node_master->start; + +$result = $node_master->psql('postgres', "COMMIT PREPARED 'gxid1'"); +is($result, 0, 'Commit foreigin transaction after crash recovery'); +$result = $node_master->psql('postgres', "ROLLBACK PREPARED 'gxid2'"); +is($result, 0, 'Rollback foreigin transaction after crash recovery'); + +# +# Commit transactions involving foreign servers and shutdown master node immediately. +# In this case, information about insertion and deletion of fdw_xact exists at only WAL. +# Check if fdw_xact entry can be processed properly during recovery. +# +$node_master->safe_psql('postgres', " +BEGIN; +UPDATE t1 SET c = 1 WHERE c = 1; +UPDATE t2 SET c = 2 WHERE c = 2; +COMMIT; +"); + +$node_master->teardown_node; +$node_master->start; + +$result = $node_master->safe_psql('postgres', 'SELECT count(*) FROM pg_fdw_xacts'); +is($result, 0, "Remove fdw_xact entry during recovery"); + +# +# A foreign server down after prepared foregin transaction but before commit it. +# Check dangling transaction can be processed propelry by pg_fdw_xact() function. +# +$node_master->safe_psql('postgres', " +BEGIN; +UPDATE t1 SET c = 1 WHERE c = 1; +UPDATE t2 SET c = 2 WHERE c = 2; +PREPARE TRANSACTION 'gxid1'; +"); + +$node_fs1->stop; + +# Since node_fs1 down COMMIT PREPARED will fail on node_fs1. +$node_master->psql('postgres', "COMMIT PREPARED 'gxid1'"); + +$node_fs1->start; +$result = $node_master->safe_psql('postgres', "SELECT count(*) FROM pg_fdw_xact_resolve() WHERE status = 'resolved'"); +is($result, 1, "pg_fdw_xact_resolve function"); + +# +# Check if the standby node can process prepared foreign transaction after +# promotion of the standby server. +# +$node_master->safe_psql('postgres', " +BEGIN; +UPDATE t1 SET c = 5 WHERE c = 5; +UPDATE t2 SET c = 6 WHERE c = 6; +PREPARE TRANSACTION 'gxid1'; +BEGIN; +UPDATE t1 SET c = 7 WHERE c = 7; +UPDATE t2 SET c = 8 WHERE c = 8; +PREPARE TRANSACTION 'gxid2'; +"); + +$node_master->teardown_node; +$node_standby->promote; + +$result = $node_standby->psql('postgres', "COMMIT PREPARED 'gxid1'"); +is($result, 0, 'Commit foreigin transaction after promotion'); +$result = $node_standby->psql('postgres', "ROLLBACK PREPARED 'gxid2'"); +is($result, 0, 'Rollback foreigin transaction after promotion'); +$result = $node_standby->safe_psql('postgres', "SELECT count(*) FROM pg_fdw_xacts"); +is($result, 0, "Check fdw_xact entry on new master node"); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index f1c1b44..6f1da60 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1323,6 +1323,13 @@ pg_cursors| SELECT c.name, c.is_scrollable, c.creation_time FROM pg_cursor() c(name, statement, is_holdable, is_binary, is_scrollable, creation_time); +pg_fdw_xacts| SELECT f.dbid, + f.transaction, + f.serverid, + f.userid, + f.status, + f.identifier + FROM pg_fdw_xacts() f(dbid, transaction, serverid, userid, status, identifier); pg_file_settings| SELECT a.sourcefile, a.sourceline, a.seqno, diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c index abb742b..506043c 100644 --- a/src/test/regress/pg_regress.c +++ b/src/test/regress/pg_regress.c @@ -2263,9 +2263,11 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc * Adjust the default postgresql.conf for regression testing. The user * can specify a file to be appended; in any case we expand logging * and set max_prepared_transactions to enable testing of prepared - * xacts. (Note: to reduce the probability of unexpected shmmax - * failures, don't set max_prepared_transactions any higher than - * actually needed by the prepared_xacts regression test.) + * xacts. We also set max_prepared_foreign_transctions to enable testing + * of atomic foreign transactions. (Note: to reduce the probability of + * unexpected shmmax failures, don't set max_prepared_transactions or + * max_prepared_foreign_transactions any higher than actually needed by the + * corresponding regression tests.). */ snprintf(buf, sizeof(buf), "%s/data/postgresql.conf", temp_instance); pg_conf = fopen(buf, "a"); @@ -2280,7 +2282,8 @@ regression_main(int argc, char *argv[], init_function ifunc, test_function tfunc fputs("log_line_prefix = '%m [%p] %q%a '\n", pg_conf); fputs("log_lock_waits = on\n", pg_conf); fputs("log_temp_files = 128kB\n", pg_conf); - fputs("max_prepared_transactions = 2\n", pg_conf); + fputs("max_prepared_transactions = 3\n", pg_conf); + fputs("max_prepared_foreign_transactions = 2\n", pg_conf); for (sl = temp_configs; sl != NULL; sl = sl->next) {