From ded270c99b3167269de1dc7dd290bfebf6abeff9 Mon Sep 17 00:00:00 2001
From: Kyotaro Horiguchi <horikyota.ntt@gmail.com>
Date: Thu, 3 Oct 2024 18:24:54 +0900
Subject: [PATCH v35 04/21] Introduce undo log implementation

This commit introduces the UNDO log system. In this implementation,
undo information is primarily stored in dynamic shared memory (DSM),
processed and dropped at transaction commit. Undo information is also
WAL-logged and restored to the same in-memory structure during
recovery.  To ensure correct recovery beyond checkpoints, undo logs in
DSM are copied at each checkpoint to files in the `pg_ulog` directory,
with filenames based on full transaction IDs. During (sub)transaction
commits, aborts, and cleanups at recovery end and consistency point,
the UNDO log system calls the undo routines (not yet implemented) with
the appropriate transaction states, allowing these routines to perform
necessary operations.
---
 src/backend/access/transam/twophase.c         |    3 +
 src/backend/access/transam/undolog.c          | 1265 +++++++++++++++++
 src/backend/access/transam/xact.c             |   21 +
 src/backend/access/transam/xlog.c             |   14 +-
 src/backend/access/transam/xlogrecovery.c     |    2 +
 src/backend/storage/ipc/ipci.c                |    3 +
 src/backend/storage/lmgr/lwlock.c             |    3 +
 .../utils/activity/wait_event_names.txt       |    1 +
 src/backend/utils/init/postinit.c             |    4 +
 src/bin/initdb/initdb.c                       |   17 +
 src/bin/pg_waldump/t/001_basic.pl             |    3 +-
 src/include/access/undolog.h                  |   22 +
 src/include/storage/lwlock.h                  |    3 +
 src/include/storage/lwlocklist.h              |    1 +
 src/tools/pgindent/typedefs.list              |    6 +
 15 files changed, 1364 insertions(+), 4 deletions(-)

diff --git a/src/backend/access/transam/twophase.c b/src/backend/access/transam/twophase.c
index 0a62c237b9d..8455ceb057a 100644
--- a/src/backend/access/transam/twophase.c
+++ b/src/backend/access/transam/twophase.c
@@ -82,6 +82,7 @@
 #include "access/transam.h"
 #include "access/twophase.h"
 #include "access/twophase_rmgr.h"
+#include "access/undolog.h"
 #include "access/xact.h"
 #include "access/xlog.h"
 #include "access/xloginsert.h"
@@ -1607,6 +1608,8 @@ FinishPreparedTransaction(const char *gid, bool isCommit)
 									   abortstats,
 									   gid);
 
+	UndoLog_UndoByXid(isCommit, xid, hdr->nsubxacts, children, false);
+
 	ProcArrayRemove(proc, latestXid);
 
 	/*
diff --git a/src/backend/access/transam/undolog.c b/src/backend/access/transam/undolog.c
index c32f5cd0b6f..d19caac5946 100644
--- a/src/backend/access/transam/undolog.c
+++ b/src/backend/access/transam/undolog.c
@@ -12,12 +12,1093 @@
  *
  * src/backend/access/transam/undolog.c
  *
+ * Each undo log record is stored in a dynamic shared area block, mapped by a
+ * dynamic shared hash. These records are WAL-logged but not immediately
+ * written to files; instead, they are flushed to multiple files at every
+ * checkpoint. Otherwise, no files are written. Unless during recovery, the
+ * xids for which a backend created undo logs are recorded in a local array for
+ * fast lookup of undo logs to process at commit.
+ *
  *-------------------------------------------------------------------------
  */
 
 #include "postgres.h"
 
+#include <sys/stat.h>
+
+#include "lib/stringinfo.h"
+#include "access/parallel.h"
 #include "access/undolog.h"
+#include "access/twophase_rmgr.h"
+#include "access/twophase.h"
+#include "access/xact.h"
+#include "access/xlog.h"
+#include "access/xloginsert.h"
+#include "lib/dshash.h"
+#include "miscadmin.h"
+#include "storage/fd.h"
+#include "storage/procarray.h"
+#include "utils/memutils.h"
+
+
+#define ULOG_FILE_MAGIC 0x474f4c55	/* 'ULOG' in big-endian */
+
+/* Resource manager definition */
+typedef struct RmgrUndoData
+{
+	const char *rm_name;
+	void	(*rm_undo) (UndoLogRecord *record, ULogOp op,
+						bool recovered, bool redo);
+	void	(*rm_undocleanupinit) (void);
+	void	(*rm_undorecoveryend) (void);
+} RmgrUndoData;
+
+#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask,decode,undo,undo_desc,undo_identify,undo_cleanupinit,undo_recoveryend) \
+	{ name, undo, undo_cleanupinit, undo_recoveryend },
+
+static RmgrUndoData	RmgrUndo[RM_MAX_ID + 1] = {
+#include "access/rmgrlist.h"
+};
+#undef PG_RMGR
+
+/*
+ * Undo log DSA block struct, each block is pointed from a dshash entry.
+ *
+ * refcount does not directly represent the number of referencers. It is set to
+ * 2 when allocated, then decremented by 2 when dropped. It is decremented by 1
+ * if the contents of this entry have been moved to another DSA entry for the
+ * same hash entry due to space expansion. Checkpointer increments the refcount
+ * while writing the file for this entry to prevent the entry from being freed.
+ *
+ * image is written by backends (or by startup during recovery), and then read
+ * by the writers and the checkpointer. written_upto primarily stores the
+ * working state of the checkpointer, and it is not crucial if the value is
+ * slightly outdated. Therefore, we do not use an LWLock and instead rely on a
+ * memory barrier to ensure the integrity of this entry.
+ */
+typedef struct UndoLogEntry
+{
+	FullTransactionId	xid;			/* target full xid */
+	pg_atomic_uint32	refcount; 		/* reference count */
+	off_t		   		written_upto;	/* how far this block is flushed */
+	bool				recovered;		/* restored in recovery ? */
+	Size		   		image_buf_len;	/* record image buffer length */
+	Size		   		image_len;		/* record image length.*/
+	unsigned char  		image[FLEXIBLE_ARRAY_MEMBER]; /* record image buffer */
+} UndoLogEntry;
+
+/* Allocation size for UndoLogEntry */
+#define UndoLogEntrySize(body_len) \
+	(offsetof(UndoLogEntry, image) + body_len)
+
+/* Initial undolog block length, arbitrary number. */
+#define InitialUndoLogLen 256
+
+/* round the size up to the nearest power of two */
+static inline Size
+undolog_adjust_bufsize(Size target)
+{
+	Size s = InitialUndoLogLen;
+
+	while (s < target)
+		s *= 2;
+
+	return s;
+}
+
+/*
+ * Initial space capacity for the record image of a newly created undo log
+ * entry
+ */
+#define UNDOLOG_INITIAL_LOG_CAPACITY 32
+
+/* Undo log dshash entry, keyed by xid */
+typedef struct UndoLogHashEntry
+{
+	FullTransactionId xid;		/* xid */
+	dsa_pointer		  body;		/* UndoLogEntry DSA pointer */
+} UndoLogHashEntry;
+
+/* Array for storing xids belongs to the current session */
+#define ACTIVE_ULOG_XIDLIST_INITIAL_LEN 32
+
+/*
+ * Struct for top-level management variables.
+ *
+ * Stored in local memory. current_ulog points to the currently active undo log
+ * dsa block, for the transaction xid. buf holds a memory area of length buflen
+ * for various uses in this module, helping to avoid frequent palloc/pfree
+ * cycles.
+ */
+typedef struct ULogStateData
+{
+	MemoryContext		cxt;			/* working memroy context */
+	UndoLogEntry	   *current_ulog;	/* current open entry */
+	dsa_area		   *dsa;			/* UndoLogEntry dsa */
+	dshash_table	   *hash;			/* UndoLogHashEntry hash */
+	FullTransactionId  *ulog_xids;		/* xid list, length ulog_xids_cap */
+	int					ulog_xids_cap;	/* capacity of the above list */
+	int					ulog_xids_num;	/* number of elements */
+	void			   *buf;			/* working buffer */
+	int					buflen;			/* length of the buffer */
+} ULogStateData;
+
+static ULogStateData ULogState = {NULL, NULL, NULL, NULL, NULL, 0, 0, NULL, 0};
+
+#define UndoLogContext (ULogState.cxt)
+
+/*
+ * Struct for bootstrap info about dynamic shared memory, stored in static
+ * shared memory.
+ */
+typedef struct UndoLogCtrlStruct
+{
+	dsa_handle			dsah;
+	dshash_table_handle dshh;
+	pg_atomic_uint32	nhashent;	/* # of hash entries in dshh */
+} UndoLogCtrlStruct;
+
+static UndoLogCtrlStruct *UndoLogCtrl;
+
+/* dshash parameter */
+static const dshash_parameters dsh_params = {
+	sizeof(FullTransactionId),
+	sizeof(UndoLogHashEntry),
+	dshash_memcmp,
+	dshash_memhash,
+	dshash_memcpy,
+	LWTRANCHE_UNDOLOG_HASH
+};
+
+/*
+ * Shared memory intializer functions
+ */
+Size
+UndoLogShmemSize(void)
+{
+	return MAXALIGN(sizeof(UndoLogCtrlStruct));
+}
+
+void
+UndoLogShmemInit(void)
+{
+	bool found;
+
+	UndoLogCtrl = (UndoLogCtrlStruct *) ShmemInitStruct("UNDO Log Data",
+														UndoLogShmemSize(),
+														&found);
+	if (!found)
+	{
+		UndoLogCtrl->dsah = DSA_HANDLE_INVALID;
+		UndoLogCtrl->dshh = DSHASH_HANDLE_INVALID;
+	}
+}
+
+/*
+ * InitUndoLog() - initialize undo log system
+ */
+void
+InitUndoLog(void)
+{
+	/* shouldn't be called from postmaster */
+	Assert(IsUnderPostmaster || !IsPostmasterEnvironment);
+
+	UndoLogContext = AllocSetContextCreate(TopMemoryContext,
+										   "Undo log system",
+										   ALLOCSET_DEFAULT_SIZES);
+
+	LWLockAcquire(UndoLogLock, LW_EXCLUSIVE);
+
+	if (UndoLogCtrl->dshh == DSHASH_HANDLE_INVALID)
+	{
+		/* We're the first process, allocate them. */
+		ULogState.dsa = dsa_create(LWTRANCHE_UNDOLOG_DSA);
+		dsa_pin(ULogState.dsa);
+		dsa_pin_mapping(ULogState.dsa);
+
+		ULogState.hash = dshash_create(ULogState.dsa, &dsh_params, NULL);
+
+		/* Share handles with succeeding processes */
+		UndoLogCtrl->dsah = dsa_get_handle(ULogState.dsa);
+		UndoLogCtrl->dshh = dshash_get_hash_table_handle(ULogState.hash);
+		pg_atomic_init_u32(&UndoLogCtrl->nhashent, 0);
+	}
+	else
+	{
+		/* Attach to existing dsm and dshash table */
+		ULogState.dsa = dsa_attach(UndoLogCtrl->dsah);
+		dsa_pin_mapping(ULogState.dsa);
+		ULogState.hash = dshash_attach(ULogState.dsa, &dsh_params,
+											UndoLogCtrl->dshh, NULL);
+	}
+
+	LWLockRelease(UndoLogLock);
+
+	/* xids list for the current session */
+	if (ULogState.ulog_xids_cap == 0)
+	{
+		Size alloc_size;
+
+		ULogState.ulog_xids_cap = UNDOLOG_INITIAL_LOG_CAPACITY;
+
+		alloc_size = ULogState.ulog_xids_cap * sizeof(FullTransactionId);
+		ULogState.ulog_xids = MemoryContextAlloc(UndoLogContext, alloc_size);
+	}
+}
+
+/*
+ * undolog_ensure_buffer()
+ *
+ * Ensures that the data buffer in ULogState is larger than the specified size.
+ */
+static void *
+undolog_ensure_buffer(Size size)
+{
+	if (size > ULogState.buflen)
+	{
+		if (likely(ULogState.buf))
+			ULogState.buf = repalloc(ULogState.buf, size);
+		else
+			ULogState.buf = MemoryContextAlloc(UndoLogContext, size);
+		ULogState.buflen = size;
+	}
+
+	return ULogState.buf;
+}
+
+/*
+ * undolog_file_exists() - Checks for file corresponding to specified xid.
+ */
+static bool
+undolog_file_exists(FullTransactionId xid)
+{
+	char fname[MAXPGPATH];
+	struct stat statbuf;
+
+	UndoLogSetFilename(fname, xid);
+
+	if (stat(fname, &statbuf) < 0)
+	{
+		if (errno == ENOENT)
+			return false;
+
+		ereport(FATAL,
+				errcode_for_file_access(),
+				errmsg("stat failed for undo file \"%s\": %m", fname));
+	}
+
+	return true;
+}
+
+/*
+ * undolog_create_entry() - Creates a new undo log memory entry
+ *
+ *  Creates a new undo log entry and hash entry for the xid, with an initial
+ *  payload of bodylen. Assumes that the hash entry for the xid does not
+ *  exist. Then stores the xid in the local xid list, unless during recovery.
+ */
+static UndoLogEntry *
+undolog_create_entry(FullTransactionId xid, Size bodylen, bool redo)
+{
+	dsa_pointer chunk;
+	UndoLogEntry *newlog;
+	UndoLogHashEntry *shhashent;
+	bool		found;
+
+	if (!redo)
+	{
+		Assert(ULogState.ulog_xids_num <= ULogState.ulog_xids_cap);
+
+		/* Expand xids array if needed */
+		if (ULogState.ulog_xids_num == ULogState.ulog_xids_cap)
+		{
+			ULogState.ulog_xids_cap *= 2;
+			ULogState.ulog_xids =
+				repalloc(ULogState.ulog_xids,
+						 ULogState.ulog_xids_cap * sizeof(FullTransactionId));
+		}
+		ULogState.ulog_xids[ULogState.ulog_xids_num++] = xid;
+	}
+
+
+	/* Adjust allocation size */
+	bodylen = undolog_adjust_bufsize(bodylen);
+
+	/* Allocate undo log memory entry with the initial size */
+	chunk = dsa_allocate0(ULogState.dsa, UndoLogEntrySize(bodylen));
+	newlog = dsa_get_address(ULogState.dsa, chunk);
+
+	newlog->xid = xid;
+	pg_atomic_init_u32(&newlog->refcount, 2);
+	newlog->written_upto = 0;
+	newlog->recovered = false;
+	newlog->image_buf_len = bodylen;
+	newlog->image_len = 0;
+
+	/* the following dshash access is exepcted to act as write barrier */
+
+	/* Register the memory entry into the hash. */
+	shhashent = dshash_find_or_insert(ULogState.hash, &xid, &found);
+	Assert(!found);
+	shhashent->body = chunk;
+	dshash_release_lock(ULogState.hash, shhashent);
+	pg_atomic_add_fetch_u32(&UndoLogCtrl->nhashent, 1);
+	return newlog;
+}
+
+/*
+ * undolog_find_ulog() - Finds the undo log entry for the xid
+ *
+ *  If the undo log for the xid is already in shared memory, return the
+ *  entry. Otherwise, load it from a file if one exists. If create is true, a
+ *  new log is created if no previous one is found.
+ */
+static UndoLogEntry *
+undolog_find_ulog(FullTransactionId xid, bool create, bool redo, bool *found)
+{
+	UndoLogHashEntry *shhashent;
+
+	*found = false;
+
+	/* fastpath for the currenlty active undo log */
+	if (ULogState.current_ulog &&
+		FullTransactionIdEquals(xid, ULogState.current_ulog->xid))
+	{
+		*found = true;
+		return ULogState.current_ulog;
+	}
+
+	/* search for the in-memory entry */
+	shhashent = dshash_find(ULogState.hash, &xid, false);
+	if (shhashent)
+	{
+		ULogState.current_ulog =
+			dsa_get_address(ULogState.dsa, shhashent->body);
+		dshash_release_lock(ULogState.hash, shhashent);
+
+		Assert(FullTransactionIdEquals(ULogState.current_ulog->xid, xid));
+
+		*found = true;
+		return ULogState.current_ulog;
+	}
+
+	/* Finally, check for the file. */
+	if (undolog_file_exists(xid))
+	{
+		char fname[MAXPGPATH];
+		struct stat sbuf;
+		int fd;
+		int ret;
+		UndoLogEntry *ulog;
+
+		UndoLogSetFilename(fname, xid);
+		fd = BasicOpenFile(fname, PG_BINARY | O_RDONLY);
+
+		if (fstat(fd, &sbuf) < 0)
+			ereport(FATAL,
+					errcode_for_file_access(),
+					errmsg("failed to stat ulog file \"%s\": %m", fname));
+
+		ulog = undolog_create_entry(xid, sbuf.st_size, redo);
+		Assert(ulog->image_buf_len >= sbuf.st_size);
+
+		ret = read(fd, ulog->image, sbuf.st_size);
+		if (ret < 0)
+			ereport(FATAL,
+					errcode_for_file_access(),
+					errmsg("failed to read ulog file \"%s\": %m", fname));
+		close(fd);
+
+		ulog->image_len = sbuf.st_size;
+
+		/* ensure all previous writes are visible before follower continues. */
+		pg_write_barrier();
+
+		ULogState.current_ulog = ulog;
+
+		*found = true;
+		return ULogState.current_ulog;
+	}
+
+	if (create)
+	{
+		xl_ulog_write *wrec;
+		UndoLogFileHeader *fheader;
+		UndoLogEntry *ulog;
+		Size		bodylen;
+		Size		wreclen;
+		XLogRecPtr	recptr;
+
+		/*
+		 * WAL-log the creation of this undo log file. However, we don't
+		 * actually crate the file since undo log is usually processed
+		 * in-memory.
+		 */
+		Assert(FullTransactionIdIsValid(xid));
+
+		if (!redo)
+		{
+			xl_ulog_create crec;
+
+			crec.xid = xid;
+			XLogBeginInsert();
+			XLogRegisterData((char *) &crec, sizeof(crec));
+			(void) XLogInsert(RM_ULOG_ID, XLOG_ULOG_CREATE);
+		}
+
+		ulog = undolog_create_entry(xid, InitialUndoLogLen, redo);
+
+		bodylen = sizeof(UndoLogFileHeader);
+		wreclen = sizeof(xl_ulog_write) + bodylen;
+		wrec = undolog_ensure_buffer(wreclen);
+
+		wrec->xid = xid;
+		wrec->off = 0;
+		wrec->len = bodylen;
+		fheader = (UndoLogFileHeader *) &wrec->bytes;
+		fheader->magic = ULOG_FILE_MAGIC;
+
+		if (!redo)
+		{
+			XLogBeginInsert();
+			XLogRegisterData((char *) wrec, wreclen);
+			recptr = XLogInsert(RM_ULOG_ID, XLOG_ULOG_WRITE);
+			XLogFlush(recptr);
+		}
+
+		Assert(ulog->image_buf_len >= bodylen);
+		memcpy(ulog->image, fheader, bodylen);
+		ulog->image_len = bodylen;
+
+		/* ensure all previous writes are visible before follower continues. */
+		pg_write_barrier();
+
+		ULogState.current_ulog = ulog;
+	}
+	else
+		ULogState.current_ulog = NULL;
+
+	return ULogState.current_ulog;
+}
+
+/*
+ * undolog_remove_file() - Removes a file specified by ULogState.file_name.
+ *
+ * The file must already have been closed.
+ */
+static void
+undolog_remove_file(FullTransactionId xid)
+{
+	char fname[MAXPGPATH];
+
+	UndoLogSetFilename(fname, xid);
+
+	durable_unlink(fname, FATAL);
+}
+
+static void
+undolog_drop_ulog(FullTransactionId xid, bool redo)
+{
+	UndoLogHashEntry *shhashent;
+	dsa_pointer ulog_p;
+	UndoLogEntry *ulog;
+
+	Assert(FullTransactionIdIsValid(xid));
+
+	/* dereference current_ulog if it is about to be dropped */
+	if (ULogState.current_ulog &&
+		FullTransactionIdEquals(xid, ULogState.current_ulog->xid))
+		ULogState.current_ulog = NULL;
+
+	/*
+	 * Search for the hash entry. Minimize the time before delete to release
+	 * the lock quickly. We should find the entry here.
+	 */
+	shhashent = dshash_find(ULogState.hash, &xid, true);
+	ulog_p = shhashent->body;
+	dshash_delete_entry(ULogState.hash, shhashent);
+	Assert(shhashent);
+
+	pg_atomic_sub_fetch_u32(&UndoLogCtrl->nhashent, 1);
+
+	/*
+	 * Remove the dsa entry, then the file if any.
+	 */
+	ulog = dsa_get_address(ULogState.dsa, ulog_p);
+
+	/*
+	 * Decrement the refcount by 2. If it reaches 0, no other process is
+	 * referencing it, so we can and should remove the DSA entry. Undo log file
+	 * can be removed in either case, as no one will read it.
+	 */
+	if (pg_atomic_sub_fetch_u32(&ulog->refcount, 2) == 0)
+		dsa_free(ULogState.dsa, ulog_p);
+
+	if (undolog_file_exists(xid))
+		undolog_remove_file(xid);
+
+	/*
+	 * Remove this xid from the local xid list if not in recovery.
+	 *
+	 * This is kind of bogus during the two-phase commit phase, as the xid list
+	 * in ULogState is empty in this case, but this function still tries to
+	 * remove the xid from the list. Since the list is empty, there's no issue
+	 * here.
+	 */
+	if (!redo)
+	{
+		for (int i = 0 ; i < ULogState.ulog_xids_num ; i++)
+		{
+			if (FullTransactionIdEquals(ULogState.ulog_xids[i], xid))
+			{
+				for (; i < ULogState.ulog_xids_num - 1; i++)
+					ULogState.ulog_xids[i] = ULogState.ulog_xids[i + 1];
+
+				ULogState.ulog_xids_num--;
+			}
+		}
+	}
+}
+
+
+/*
+ * undolog_write_to_file - write the undo log into file
+ *
+ *   Writes the undo log image from start_off to end_off.
+ */
+static void
+undolog_write_to_file(UndoLogEntry *ulog, off_t start_off, off_t end_off)
+{
+	char fname[MAXPGPATH];
+	int fd;
+	int ret;
+	Size	write_len;
+
+	Assert(end_off > start_off);
+
+	write_len = end_off - start_off;
+
+	if (write_len == 0)
+		return;
+
+	UndoLogSetFilename(fname, ulog->xid);
+	fd = BasicOpenFile(fname, PG_BINARY | O_WRONLY | O_CREAT);
+	if (fd < 0)
+		ereport(FATAL,
+				errcode_for_file_access(),
+				errmsg("failed to open or create undo file \"%s\": %m", fname));
+
+	ret = pg_pwrite(fd,
+					ulog->image + start_off,	/* buf address */
+					write_len,					/* write_size */
+					start_off);					/* file offset */
+
+	if (ret != write_len)
+		ereport(FATAL,
+				errcode_for_file_access(),
+				errmsg("failed to write to undo file \"%s\": %m", fname));
+
+	close(fd);
+
+	return;
+}
+
+static UndoLogEntry *
+undolog_realloc_ulog(UndoLogEntry *ulog, Size target_size)
+{
+	UndoLogHashEntry *shhashent;
+	dsa_pointer chunk;
+	dsa_pointer oldchunk;
+	Size		oldsize;
+	Size		newsize;
+	UndoLogEntry *newlog;
+	UndoLogEntry *oldlog = ulog;
+
+	if (ulog->image_buf_len >= target_size)
+		return oldlog;
+
+	/* adjust the target size */
+	target_size = undolog_adjust_bufsize(target_size);
+
+	/*
+	 * Allocate a new undo log entry with double the size, then copy its
+	 * contents. No other process will write to the old entry, so there¢s no
+	 * need to lock it.
+	 */
+	oldsize = UndoLogEntrySize(oldlog->image_buf_len);
+	newsize = UndoLogEntrySize(target_size);
+	chunk = dsa_allocate0(ULogState.dsa, newsize);
+	newlog = dsa_get_address(ULogState.dsa, chunk);
+
+	/* don't bother clearing expanded area */
+	memcpy(newlog, oldlog, oldsize);
+
+	/* adjust and initialize some attributes of the new log */
+	newlog->image_buf_len = target_size;
+	pg_atomic_init_u32(&newlog->refcount, 2);
+
+	ULogState.current_ulog = ulog = newlog;
+
+	/*
+	 * Remap the hash entry to point to the new body. This shared hash is
+	 * likely to become a hotspot, so be careful not to hold the lock for
+	 * too long.
+	 */
+	shhashent = dshash_find(ULogState.hash, &ulog->xid, true);
+	oldchunk = shhashent->body;
+	shhashent->body = chunk;
+	dshash_release_lock(ULogState.hash, shhashent);
+
+	/*
+	 * Mark the entry body as moved, and free it if no other process is
+	 * working on it. Otherwise, let that process handle the task.
+	 */
+	Assert(oldlog == dsa_get_address(ULogState.dsa, oldchunk));
+	if (pg_atomic_sub_fetch_u32(&oldlog->refcount, 1) == 1)
+	{
+		/*
+		 * No other process is referencing this entry. Decrement again
+		 */
+		if (pg_atomic_sub_fetch_u32(&oldlog->refcount, 1) == 0)
+			dsa_free(ULogState.dsa, oldchunk);
+	}
+
+	return newlog;
+}
+
+/*
+ * UndoLogWrite() - Writes an undolog record using current xid
+ *
+ * This write is WAL-logged.
+ */
+void
+UndoLogWrite(RmgrId rmgr, uint8 info, void *data, int len)
+{
+	FullTransactionId xid;
+	int reclen = sizeof(UndoLogRecord) + len;
+	int wreclen = sizeof(xl_ulog_write) + reclen;
+	xl_ulog_write  *wrec;
+	UndoLogRecord  *rec;
+	pg_crc32c		undodata_crc;
+	UndoLogEntry   *ulog;
+	XLogRecPtr		recptr;
+	bool			found;
+
+	Assert(!RecoveryInProgress());
+	Assert(!IsParallelWorker());
+
+	/*
+	 *  The following line may assign a new transaction ID. This is somewhat
+	 *  clumsy, but the caller needs to assign it soon.
+	 */
+	xid = GetCurrentFullTransactionId();
+
+	if (!IsUnderPostmaster)
+		return;
+
+	/* the caller can set rmgr bits only */
+	Assert((info & ~ULR_RMGR_INFO_MASK) == 0);
+
+	ulog = undolog_find_ulog(xid, true, false, &found);
+	Assert(ulog);
+
+	/* create undo record as a part of WAL record to avoid copying */
+	wrec = undolog_ensure_buffer(wreclen);
+	rec = (UndoLogRecord *) &wrec->bytes;
+	rec->ul_tot_len = reclen;
+	rec->ul_rmid = rmgr;
+	rec->ul_info = info;
+
+	memcpy((char *)rec + sizeof(UndoLogRecord), data, len);
+
+	/* calculate CRC of the data */
+	INIT_CRC32C(undodata_crc);
+	COMP_CRC32C(undodata_crc, &rec->ul_rmid,
+				reclen - offsetof(UndoLogRecord, ul_rmid));
+	rec->ul_crc = undodata_crc;
+
+	/*
+	 * Write an XLOG record for this undo log record. It is crucial to flush
+	 * immediately, as this record will cancel the action taken immediately
+	 * after.
+	 */
+	wrec->xid = ulog->xid;
+	wrec->off = ulog->image_len;
+	wrec->len = reclen;
+	XLogBeginInsert();
+	XLogRegisterData((char *) wrec, wreclen);
+	recptr = XLogInsert(RM_ULOG_ID, XLOG_ULOG_WRITE);
+	XLogFlush(recptr);
+
+	/* Write the undo log record. */
+
+	/* expand entry body if it is too short */
+	if (ulog->image_buf_len < ulog->image_len + reclen)
+		ulog = undolog_realloc_ulog(ulog, ulog->image_len + reclen);
+
+	/* Append the record. */
+	memcpy(ulog->image + ulog->image_len, rec, reclen);
+	ulog->image_len += reclen;
+
+	/* ensure all previous writes are visible before follower continues. */
+	pg_write_barrier();
+}
+
+/*
+ * ulog_process_ulog() - Processes the undo log.
+ *
+ * 'op' specifies the operation mode (commit, abort, or prepare) passed to the
+ * rmgr routines. 'outercxt' is the memory context used while the rmgr routines
+ * are running.
+ */
+static void
+undolog_process_ulog(ULogOp op, UndoLogEntry *ulog,
+					 bool redo, MemoryContext outercxt)
+{
+	unsigned char *p = ulog->image;
+	unsigned char *end = p + ulog->image_len;
+
+	Assert (ulog);
+	Assert (outercxt);
+
+	p += sizeof(UndoLogFileHeader);
+
+
+	while (p < end)
+	{
+		UndoLogRecord *rec = (UndoLogRecord *)p;
+		MemoryContext	oldcxt;
+		pg_crc32c	undodata_crc;
+
+		/* CRC check */
+		INIT_CRC32C(undodata_crc);
+		COMP_CRC32C(undodata_crc, &rec->ul_rmid,
+					rec->ul_tot_len - offsetof(UndoLogRecord, ul_rmid));
+		if (!EQ_CRC32C(rec->ul_crc, undodata_crc))
+			ereport(FATAL,
+					errcode_for_file_access(),
+					errmsg("incorrect undolog record checksum at %lld for xid %lld, abort undo",
+						   (long long int) (p - (unsigned char *)ulog),
+						   (long long unsigned int) U64FromFullTransactionId(ulog->xid)));
+
+		/* The undo routines may want to allcoate memory in the outer context */
+		oldcxt = MemoryContextSwitchTo(outercxt);
+		RmgrUndo[rec->ul_rmid].rm_undo(rec, op, ulog->recovered, redo);
+		MemoryContextSwitchTo(oldcxt);
+
+		p += rec->ul_tot_len;
+	}
+}
+
+/*
+ * ulog_undo() - Processes undo log for the specified xid.
+ *
+ * The undo log file for the xid is removed before this function returns,
+ * regardless whether it is processed or not. Therefore, the in-memory entry
+ * for this xid must be removed afterwards, if any.
+ */
+static void
+undolog_undo(bool isCommit, UndoLogEntry *ulog, FullTransactionId xid,
+			 bool redo)
+{
+	ULogOp op;
+
+	Assert(!IsParallelWorker());
+
+	if (!ulog)
+		return;
+
+	if (isCommit)
+		op = ULOGOP_COMMIT;
+	else
+		op = ULOGOP_ABORT;
+
+	undolog_process_ulog(op, ulog, redo, CurrentMemoryContext);
+
+	undolog_drop_ulog(xid, redo);
+}
+
+/*
+ * UndoLog_UndoByXid()
+ *
+ * Processes undo logs for the specified transactions, intended for use in
+ * finishing prepared transactions or during recovery.
+ *
+ * children is the list of subtransaction IDs of the xid, with a length of
+ * nchildren.
+ */
+void
+UndoLog_UndoByXid(bool isCommit, TransactionId xid,
+						int nchildren, TransactionId *children, bool redo)
+{
+	uint32				nextepoch;
+	TransactionId		nextxid;
+	uint32 				epoch;
+	FullTransactionId	fxid;
+	UndoLogEntry	   *ulog;
+	bool				found;
+
+	nextepoch = EpochFromFullTransactionId(TransamVariables->nextXid);
+	nextxid = XidFromFullTransactionId(TransamVariables->nextXid);
+
+	/* Adjust epoch, if needed. */
+	if (xid <= nextxid)
+		epoch = nextepoch;
+	else
+		epoch = nextepoch - 1;
+
+	/* process undo logs */
+	fxid = FullTransactionIdFromEpochAndXid(epoch, xid);
+
+	ulog = undolog_find_ulog(fxid, false, redo, &found);
+
+	if (ulog)
+		undolog_undo(isCommit, ulog, fxid, redo);
+
+	for (int i = 0 ; i < nchildren ; i++)
+	{
+		if (children[i] <= nextxid)
+			epoch = nextepoch;
+		else
+			epoch = nextepoch - 1;
+
+		fxid = FullTransactionIdFromEpochAndXid(epoch, children[i]);
+
+		ulog = undolog_find_ulog(fxid, false, redo, &found);
+
+		if (ulog)
+			undolog_undo(isCommit, ulog, fxid, redo);
+	}
+}
+
+/*
+ * AtEOXact_UndoLog() - At end-of-xact processing of undo logs.
+ *
+ * Processes all existing undo log files, leaving none remaining after this
+ * function returns.
+ */
+void
+AtEOXact_UndoLog(bool isCommit)
+{
+	if (ULogState.ulog_xids_num == 0)
+		return;
+
+	for (int i = ULogState.ulog_xids_num - 1 ; i >= 0 ; i--)
+	{
+		UndoLogEntry   *ulog;
+		bool			found;
+
+		ulog = undolog_find_ulog(ULogState.ulog_xids[i], false, false, &found);
+		Assert(ulog);
+
+		undolog_undo(isCommit, ulog, ULogState.ulog_xids[i], false);
+	}
+
+	ULogState.current_ulog = NULL;
+}
+
+/*
+ * AtEOXact_UndoLog() - At end-of-subxact processing of undo logs.
+ *
+ * The undo log for the subtransaction will be removed on abort. It will remain
+ * on commit and be processed at the end of the top-level transaction.
+ */
+void
+AtEOSubXact_UndoLog(bool isCommit)
+{
+	FullTransactionId	xid;
+	UndoLogEntry	   *ulog;
+	bool				found;
+
+	/*
+	 * Undo logs of committed subtransactions are processed at the end of the
+	 * top-level transaction.
+	 */
+	if (isCommit)
+		return;
+
+	xid = GetCurrentFullTransactionIdIfAny();
+
+	/* Return if the innermost subxid is not assigned. */
+	if (!FullTransactionIdIsValid(xid))
+		return;
+
+	ulog = undolog_find_ulog(xid, false, false, &found);
+
+	if (ulog)
+		undolog_undo(isCommit, ulog, xid, false);
+}
+
+/*
+ * AtPrepare_UndoLog()
+ *
+ * Blow away the xid list for the current transaction.
+ *
+ * Undo log entries in shared memory are left as-is, waiting for
+ * finish-prepared processing.
+ */
+void
+AtPrepare_UndoLog(void)
+{
+	ULogState.ulog_xids_num = 0;
+}
+
+static void
+undolog_cleanup_init(void)
+{
+	for (int rmid = 0; rmid <= RM_MAX_ID; rmid++)
+	{
+		if (RmgrUndo[rmid].rm_name == NULL)
+			continue;
+
+		if (RmgrUndo[rmid].rm_undocleanupinit != NULL)
+			RmgrUndo[rmid].rm_undocleanupinit();
+	}
+}
+
+void
+UndoLogRecoveryEnd(void)
+{
+	for (int rmid = 0; rmid <= RM_MAX_ID; rmid++)
+	{
+		if (RmgrUndo[rmid].rm_name == NULL)
+			continue;
+
+		if (RmgrUndo[rmid].rm_undorecoveryend != NULL)
+			RmgrUndo[rmid].rm_undorecoveryend();
+	}
+}
+
+/*
+ * UndoLogCleanup() - On-recovery cleanup of undo log
+ *
+ * This function is called after ULOG file consistency is established, either
+ * when recovery reaches consistency or after recovery finishes if hot standby
+ * is not active.
+ */
+void
+UndoLogCleanup(bool end_of_recovery)
+{
+	dshash_seq_status hstat;
+	UndoLogHashEntry *p;
+	UndoLogEntry *ulog;
+	dsa_pointer ulog_dsap;
+	MemoryContext outercxt;
+
+
+	/*
+	 * Some memory allocation occurs during this process. Use a separate memory
+	 * context to avoid memory leaks.
+	 */
+	outercxt = MemoryContextSwitchTo(UndoLogContext);
+
+	undolog_cleanup_init();
+
+	/*
+	 * scan through all undo log files.
+	 *
+	 * Since we're in recovery and this is called only once, take a simpler way
+	 * in exchange for allowing a possible brief pause of checkpointer. Because
+	 * we may drop some entries, we run the dshash loop in exclusive lock mode.
+	 */
+	dshash_seq_init(&hstat, ULogState.hash, true);
+	while ((p = dshash_seq_next(&hstat)) != NULL)
+	{
+		FullTransactionId	log_fxid;
+		TransactionId		log_xid;
+		FullTransactionId	next_fxid;
+		TransactionId		next_xid;
+		uint32				oldest_epoch;
+		TransactionId		oldest_xid;
+		FullTransactionId	oldest_fxid;
+		ULogOp	op;
+		bool	xact_prepared;
+		uint32	refcount PG_USED_FOR_ASSERTS_ONLY;
+
+		ulog_dsap = p->body;
+		ulog = dsa_get_address(ULogState.dsa, ulog_dsap);
+
+		/* increment refcount to avoid deleteion of this block */
+		refcount = pg_atomic_add_fetch_u32(&ulog->refcount, 1);
+
+		/* refcount should not be 1, see CheckPointUndoLog for details. */
+		Assert(refcount > 1);
+
+		/*
+		 *  Make sure the log's xid is valid.
+		 */
+		log_fxid = ulog->xid;
+		log_xid = XidFromFullTransactionId(log_fxid);
+
+		LWLockAcquire(XactTruncationLock, LW_SHARED);
+		next_fxid = TransamVariables->nextXid;
+		oldest_xid = TransamVariables->oldestClogXid;
+		LWLockRelease(XactTruncationLock);
+
+		/* Generate full xid for oldest_xid based on next_fxid */
+		next_xid = XidFromFullTransactionId(next_fxid);
+		oldest_epoch = EpochFromFullTransactionId(next_fxid);
+
+		/* adjust epoch for oldest xid */
+		if (oldest_xid > next_xid)
+			oldest_epoch--;
+
+		oldest_fxid =
+			FullTransactionIdFromEpochAndXid(oldest_epoch, oldest_xid);
+
+		/* check the ulog xid */
+		if (FullTransactionIdPrecedes(log_fxid, oldest_fxid))
+			ereport(FATAL,
+					errcode_for_file_access(),
+					errmsg("undolog found for too-old transaction %llu",
+						   (long long unsigned int) U64FromFullTransactionId(log_fxid)));
+
+		/* All transactions with undo log must be in-progress. */
+		if (!TransactionIdIsInProgress(log_xid))
+			ereport(FATAL,
+					errcode_for_file_access(),
+					errmsg("undolog found for non-acitve transaction: %llu",
+						   (long long unsigned int) U64FromFullTransactionId(log_fxid)));
+
+		/*
+		 * Let undo routines perform cleanup tasks with appropriate
+		 * assumptions.  If the transaction is prepared or when recovery is
+		 * reaching consistency, assume it is active; otherwise, perform abort
+		 * cleanups.
+		 */
+		xact_prepared = TwoPhaseXidExists(log_xid, false);
+
+		if (!end_of_recovery || xact_prepared)
+			op = ULOGOP_ACTIVE;
+		else
+			op = ULOGOP_ABORT;
+
+		undolog_process_ulog(op, ulog, true, outercxt);
+
+		if (!xact_prepared)
+		{
+			/*
+			 * Since undolog_drop_ulog() is not supposed to be called inside
+			 * dshash loops, we deliberately replicate part of the function
+			 * here.
+			 */
+			dshash_delete_current(&hstat);
+			pg_atomic_sub_fetch_u32(&UndoLogCtrl->nhashent, 1);
+
+			if (pg_atomic_sub_fetch_u32(&ulog->refcount, 2) == 0)
+				dsa_free(ULogState.dsa, ulog_dsap);
+
+			if (undolog_file_exists(log_fxid))
+				undolog_remove_file(log_fxid);
+		}
+	}
+
+	dshash_seq_term(&hstat);
+
+	MemoryContextSwitchTo(outercxt);
+
+	ULogState.current_ulog = NULL;
+}
 
 /*
  * undollg_redo()
@@ -31,8 +1112,192 @@ undolog_redo(XLogReaderState *record)
 
 	if (info == XLOG_ULOG_CREATE)
 	{
+		xl_ulog_create	   *rec = (xl_ulog_create *) XLogRecGetData(record);
+		bool				found;
+
+		/*
+		 * We don't check for the existence of the log. Although the log should
+		 * not be found in a consistent state, it may appear during the
+		 * inconsistent period in recovery.
+		 */
+		(void) undolog_find_ulog(rec->xid, true, true, &found);
 	}
 	else if (info == XLOG_ULOG_WRITE)
 	{
+		UndoLogEntry   *ulog;
+		Size			target_size;
+		xl_ulog_write  *rec = (xl_ulog_write *) XLogRecGetData(record);
+		bool			found;
+
+		ulog = undolog_find_ulog(rec->xid, false, false, &found);
+		Assert(ulog);
+
+		/* mark this log as recovered */
+		ulog->recovered = true;
+
+		target_size = rec->off + rec->len;
+
+		if (ulog->image_buf_len < target_size)
+			ulog = undolog_realloc_ulog(ulog, target_size);
+
+		memcpy(ulog->image + rec->off, rec->bytes, rec->len);
+
+		/*
+		 * The log image can extend beyond the end of the write during crash
+		 * recovery. Even in that case, the undo log grows to its final length
+		 * until consistency is reached. Therefore, we don't perform sanity
+		 * checks on the image length. Additionally, XLOG_ULOG_CREATE writes an
+		 * extra header part, which reappears as XLOG_ULOG_WRITE. However, this
+		 * is not an issue from an integrity perspective. Although we could
+		 * omit issuing the XLOG_ULOG_WRITE record for the header part, it is
+		 * currently still emitted to maintain the apparent integrity among
+		 * XLOG records.
+		 */
+		ulog->image_len = rec->off + rec->len;
+
+		/* ensure all previous writes are visible before follower continues. */
+		pg_write_barrier();
 	}
 }
+
+/*
+ * CheckPointRelationMap
+ *
+ * This is called during a checkpoint.  It must ensure that any undo log writes
+ * that were WAL-logged before the start of the checkpoint are securely flushed
+ * to disk so that we won't lose their existence and content before this
+ * checkpoint start.
+ */
+void
+CheckPointUndoLog(void)
+{
+	dshash_seq_status hstat;
+	UndoLogHashEntry *p;
+	dsa_pointer *ulogs;
+	UndoLogEntry *ulog;
+	dsa_pointer ulog_dsap;
+	int			ulogs_len;
+	int			n_initial_ulogs;
+	int			nulogs;
+	bool		written = false;
+
+	if (!IsUnderPostmaster)
+		return;
+
+
+	/*
+	 * Allocate working area with the approximate size of the current hash
+	 * entries. This is not accurate but enough to avoid freuqent repallocs.
+	 * Add 16 as an arbitrary number to avoid repalloc as much as possible.
+	 */
+	n_initial_ulogs = pg_atomic_read_u32(&UndoLogCtrl->nhashent) + 16;
+
+	ulogs_len = 64;
+
+	while (ulogs_len < n_initial_ulogs)
+		ulogs_len *= 2;
+
+	ulogs = MemoryContextAlloc(UndoLogContext,
+							   ulogs_len * sizeof(dsa_pointer));
+	nulogs = 0;
+
+	/*
+	 * This loop acquires a lock on the dshash. To minimize lock time, we only
+	 * collect candidate UNDO logs here, and the actual processing is done
+	 * afterward.
+
+	 */
+	dshash_seq_init(&hstat, ULogState.hash, false);
+	while ((p = dshash_seq_next(&hstat)) != NULL)
+	{
+		uint32 refcount PG_USED_FOR_ASSERTS_ONLY;
+
+		ulog_dsap = p->body;
+
+		ulog = dsa_get_address(ULogState.dsa, ulog_dsap);
+
+		/* increment refcount to avoid deleteion of this block */
+		refcount = pg_atomic_add_fetch_u32(&ulog->refcount, 1);
+
+		/*
+		 * Since undolog_drop_ulog() drops the hash entry before the DSA entry,
+		 * we should not expect to fetch the dropped DSA entries at this point.
+		 * (This suggests that a read-after-write error might be occurring.)
+		 */
+		Assert(refcount > 1);
+		Assert(nulogs <= ulogs_len);
+
+		if (unlikely(nulogs == ulogs_len))
+		{
+			ulogs_len *= 2;
+			ulogs = repalloc(ulogs, ulogs_len * sizeof(UndoLogEntry *));
+		}
+
+		ulogs[nulogs] = ulog_dsap;
+		nulogs++;
+	}
+	dshash_seq_term(&hstat);
+
+	for (int i = 0 ; i < nulogs ; i++)
+	{
+		uint32 refcount;
+
+		ulog = dsa_get_address(ULogState.dsa, ulogs[i]);
+
+		/*
+		 * Check the refcount again to check if the undo log is already dropped
+		 * before actually writing the file. Since we have incremented it, 1
+		 * indicates that the entry is being dropped by any other
+		 * process. Otherwise we can safely write the contents to file.  See
+		 * UndoLogWrite() and undolog_drop_ulog() for the corresponding code.
+		 */
+		if (pg_atomic_read_u32(&ulog->refcount) > 1 &&
+			ulog->image_len > ulog->written_upto)
+		{
+			off_t start_off;
+			off_t end_off;
+
+			ulog = dsa_get_address(ULogState.dsa, ulogs[i]);
+
+			/*
+			 * Ensure visibility of previously written data before proceeding
+			 * with reads.
+			 */
+			pg_read_barrier();
+
+			/*
+			 * Copy values from shared memory to local memory to finalize them.
+			 */
+			start_off = ulog->written_upto;
+			end_off	= ulog->image_len;
+
+			if (start_off < end_off)
+			{
+				undolog_write_to_file(ulog, start_off, end_off);
+
+				if (ulog->written_upto < end_off)
+					written = true;
+
+				/* No fencing needed, as only checkpointer accesses it. */
+				ulog->written_upto = end_off;
+			}
+		}
+
+		/*
+		 * Decrement the 1 we added ourselves. If this results in 0, it means
+		 * the DSA area is no longer needed, and we ourselves need to delete
+		 * it.
+		 */
+		refcount = pg_atomic_sub_fetch_u32(&ulog->refcount, 1);
+
+		/* Free the entry if it is no longer used */
+		if (refcount < 2)
+			dsa_free(ULogState.dsa, ulogs[i]);
+	}
+
+	pfree(ulogs);
+
+	/* Sync the directory if any files have been written to it. */
+	if (written)
+		fsync_fname(UNDOLOG_DIR, true);
+}
diff --git a/src/backend/access/transam/xact.c b/src/backend/access/transam/xact.c
index 004f7e10e55..5739ef3b7f5 100644
--- a/src/backend/access/transam/xact.c
+++ b/src/backend/access/transam/xact.c
@@ -26,6 +26,7 @@
 #include "access/subtrans.h"
 #include "access/transam.h"
 #include "access/twophase.h"
+#include "access/undolog.h"
 #include "access/xact.h"
 #include "access/xlog.h"
 #include "access/xloginsert.h"
@@ -2432,6 +2433,12 @@ CommitTransaction(void)
 
 	AtEOXact_MultiXact();
 
+	/*
+	 * Discard UNDO log. This does not necessarily need to be done here, but it
+	 * is performed in the same place as abort.
+	 */
+	AtEOXact_UndoLog(true);
+
 	ResourceOwnerRelease(TopTransactionResourceOwner,
 						 RESOURCE_RELEASE_LOCKS,
 						 true, true);
@@ -2670,6 +2677,7 @@ PrepareTransaction(void)
 	AtPrepare_PgStat();
 	AtPrepare_MultiXact();
 	AtPrepare_RelationMap();
+	AtPrepare_UndoLog();
 
 	/*
 	 * Here is where we really truly prepare.
@@ -2971,6 +2979,13 @@ AbortTransaction(void)
 		AtEOXact_TypeCache();
 		AtEOXact_Inval(false);
 		AtEOXact_MultiXact();
+
+		/*
+		 * Drop storage files. This has to happen after buffer pins are
+		 * dropped, required by DropRelationBuffers().
+		 */
+		AtEOXact_UndoLog(false);
+
 		ResourceOwnerRelease(TopTransactionResourceOwner,
 							 RESOURCE_RELEASE_LOCKS,
 							 false, true);
@@ -5174,6 +5189,7 @@ CommitSubTransaction(void)
 	AtEOSubXact_TypeCache();
 	AtEOSubXact_Inval(true);
 	AtSubCommit_smgr();
+	AtEOSubXact_UndoLog(true);
 
 	/*
 	 * The only lock we actually release here is the subtransaction XID lock.
@@ -5356,6 +5372,7 @@ AbortSubTransaction(void)
 							 RESOURCE_RELEASE_AFTER_LOCKS,
 							 false, false);
 		AtSubAbort_smgr();
+		AtEOSubXact_UndoLog(false);
 
 		AtEOXact_GUC(false, s->gucNestLevel);
 		AtEOSubXact_SPI(false, s->subTransactionId);
@@ -6246,6 +6263,8 @@ xact_redo_commit(xl_xact_parsed_commit *parsed,
 		DropRelationFiles(parsed->xlocators, parsed->nrels, true);
 	}
 
+	UndoLog_UndoByXid(true, xid, parsed->nsubxacts, parsed->subxacts, true);
+
 	if (parsed->nstats > 0)
 	{
 		/* see equivalent call for relations above */
@@ -6357,6 +6376,8 @@ xact_redo_abort(xl_xact_parsed_abort *parsed, TransactionId xid,
 		DropRelationFiles(parsed->xlocators, parsed->nrels, true);
 	}
 
+	UndoLog_UndoByXid(false, xid, parsed->nsubxacts, parsed->subxacts, true);
+
 	if (parsed->nstats > 0)
 	{
 		/* see equivalent call for relations above */
diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index e9d029ebfac..5ba4468f52d 100644
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -55,6 +55,7 @@
 #include "access/timeline.h"
 #include "access/transam.h"
 #include "access/twophase.h"
+#include "access/undolog.h"
 #include "access/xact.h"
 #include "access/xlog_internal.h"
 #include "access/xlogarchive.h"
@@ -5905,13 +5906,19 @@ StartupXLOG(void)
 	if (InRecovery)
 	{
 		/*
-		 * Clean up unlogged relations if not already done. If consistency has
-		 * been established, this cleanup would have occurred when entering hot
-		 * standby mode (see CheckRecoveryConsistency for details).
+		 * If consistency has not been established, process undo log files to
+		 * clean up storage files from unfinished transactions and clean up
+		 * unlogged relations. (See CheckRecoveryConsistency for details.)
 		 */
 		if (!reachedConsistency)
+		{
+			UndoLogCleanup(true);
 			ResetUnloggedRelations(UNLOGGED_RELATION_CLEANUP);
+		}
+
 		ResetUnloggedRelations(UNLOGGED_RELATION_INIT);
+
+		UndoLogRecoveryEnd();
 	}
 
 	/*
@@ -7516,6 +7523,7 @@ CheckPointGuts(XLogRecPtr checkPointRedo, int flags)
 	CheckPointSnapBuild();
 	CheckPointLogicalRewriteHeap();
 	CheckPointReplicationOrigin();
+	CheckPointUndoLog();
 
 	/* Write out all dirty data in SLRUs and the main buffer pool */
 	TRACE_POSTGRESQL_BUFFER_CHECKPOINT_START(flags);
diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c
index 278154ad9a0..93d56dee75e 100644
--- a/src/backend/access/transam/xlogrecovery.c
+++ b/src/backend/access/transam/xlogrecovery.c
@@ -33,6 +33,7 @@
 
 #include "access/timeline.h"
 #include "access/transam.h"
+#include "access/undolog.h"
 #include "access/xact.h"
 #include "access/xlog_internal.h"
 #include "access/xlogarchive.h"
@@ -2278,6 +2279,7 @@ CheckRecoveryConsistency(void)
 		 * backends don't try to read whatever garbage is left over from
 		 * before.
 		 */
+		UndoLogCleanup(false);
 		ResetUnloggedRelations(UNLOGGED_RELATION_CLEANUP);
 
 		SpinLockAcquire(&XLogRecoveryCtl->info_lck);
diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c
index d68aa29d93e..4982861b624 100644
--- a/src/backend/storage/ipc/ipci.c
+++ b/src/backend/storage/ipc/ipci.c
@@ -22,6 +22,7 @@
 #include "access/syncscan.h"
 #include "access/transam.h"
 #include "access/twophase.h"
+#include "access/undolog.h"
 #include "access/xlogprefetcher.h"
 #include "access/xlogrecovery.h"
 #include "access/xlogwait.h"
@@ -150,6 +151,7 @@ CalculateShmemSize(int *num_semaphores)
 	size = add_size(size, InjectionPointShmemSize());
 	size = add_size(size, SlotSyncShmemSize());
 	size = add_size(size, WaitLSNShmemSize());
+	size = add_size(size, UndoLogShmemSize());
 
 	/* include additional requested shmem from preload libraries */
 	size = add_size(size, total_addin_request);
@@ -287,6 +289,7 @@ CreateOrAttachShmemStructs(void)
 	XLogPrefetchShmemInit();
 	XLogRecoveryShmemInit();
 	CLOGShmemInit();
+	UndoLogShmemInit();
 	CommitTsShmemInit();
 	SUBTRANSShmemInit();
 	MultiXactShmemInit();
diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c
index db6ed784ab3..8a12a977c4e 100644
--- a/src/backend/storage/lmgr/lwlock.c
+++ b/src/backend/storage/lmgr/lwlock.c
@@ -166,6 +166,9 @@ static const char *const BuiltinTrancheNames[] = {
 	[LWTRANCHE_SUBTRANS_SLRU] = "SubtransSLRU",
 	[LWTRANCHE_XACT_SLRU] = "XactSLRU",
 	[LWTRANCHE_PARALLEL_VACUUM_DSA] = "ParallelVacuumDSA",
+	[LWTRANCHE_UNDOLOG_DSA] = "UndoLogDSA",
+	[LWTRANCHE_UNDOLOG_HASH] = "UndoLogHash",
+	[LWTRANCHE_UNDOLOG_DATA] = "UndoLogData",
 };
 
 StaticAssertDecl(lengthof(BuiltinTrancheNames) ==
diff --git a/src/backend/utils/activity/wait_event_names.txt b/src/backend/utils/activity/wait_event_names.txt
index 8efb4044d6f..b8f31db46aa 100644
--- a/src/backend/utils/activity/wait_event_names.txt
+++ b/src/backend/utils/activity/wait_event_names.txt
@@ -347,6 +347,7 @@ DSMRegistry	"Waiting to read or update the dynamic shared memory registry."
 InjectionPoint	"Waiting to read or update information related to injection points."
 SerialControl	"Waiting to read or update shared <filename>pg_serial</filename> state."
 WaitLSN	"Waiting to read or update shared Wait-for-LSN state."
+UndoLog	"Waiting to read or update shared UNDO log state."
 
 #
 # END OF PREDEFINED LWLOCKS (DO NOT CHANGE THIS LINE)
diff --git a/src/backend/utils/init/postinit.c b/src/backend/utils/init/postinit.c
index a024b1151d0..fda74d7af00 100644
--- a/src/backend/utils/init/postinit.c
+++ b/src/backend/utils/init/postinit.c
@@ -24,6 +24,7 @@
 #include "access/htup_details.h"
 #include "access/session.h"
 #include "access/tableam.h"
+#include "access/undolog.h"
 #include "access/xact.h"
 #include "access/xlog.h"
 #include "access/xloginsert.h"
@@ -640,6 +641,9 @@ BaseInit(void)
 	 */
 	InitXLogInsert();
 
+	/* Initialize undo log system */
+	InitUndoLog();
+
 	/* Initialize lock manager's local structs */
 	InitLockManagerAccess();
 
diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c
index 9a91830783e..7c1f4e1f53a 100644
--- a/src/bin/initdb/initdb.c
+++ b/src/bin/initdb/initdb.c
@@ -307,6 +307,7 @@ void		setup_signals(void);
 void		setup_text_search(void);
 void		create_data_directory(void);
 void		create_xlog_or_symlink(void);
+void		create_ulog(void);
 void		warn_on_mount_point(int error);
 void		initialize_data_directory(void);
 
@@ -3012,6 +3013,21 @@ create_xlog_or_symlink(void)
 	free(subdirloc);
 }
 
+/* Create undo log directory */
+void
+create_ulog(void)
+{
+	char	   *subdirloc;
+
+	/* form name of the place for the subdirectory */
+	subdirloc = psprintf("%s/pg_ulog", pg_data);
+
+	if (mkdir(subdirloc, pg_dir_create_mode) < 0)
+		pg_fatal("could not create directory \"%s\": %m",
+				 subdirloc);
+
+	free(subdirloc);
+}
 
 void
 warn_on_mount_point(int error)
@@ -3046,6 +3062,7 @@ initialize_data_directory(void)
 	create_data_directory();
 
 	create_xlog_or_symlink();
+	create_ulog();
 
 	/* Create required subdirectories (other than pg_wal) */
 	printf(_("creating subdirectories ... "));
diff --git a/src/bin/pg_waldump/t/001_basic.pl b/src/bin/pg_waldump/t/001_basic.pl
index 578e4731394..09396f065fa 100644
--- a/src/bin/pg_waldump/t/001_basic.pl
+++ b/src/bin/pg_waldump/t/001_basic.pl
@@ -73,7 +73,8 @@ BRIN
 CommitTs
 ReplicationOrigin
 Generic
-LogicalMessage$/,
+LogicalMessage
+UndoLog$/,
 	'rmgr list');
 
 
diff --git a/src/include/access/undolog.h b/src/include/access/undolog.h
index 8955197398d..827d3f9fa90 100644
--- a/src/include/access/undolog.h
+++ b/src/include/access/undolog.h
@@ -33,6 +33,14 @@ typedef struct UndoLogRecord
 	/* rmgr-specific data follow, no padding */
 } UndoLogRecord;
 
+/* Operation mode for rm_uedo() resource manger routine */
+typedef enum ULogOp
+{
+	ULOGOP_COMMIT,		/* tell to perform commit action */
+	ULOGOP_ABORT,		/* tell to perform abort action */
+	ULOGOP_ACTIVE		/* tell xact is active */
+} ULogOp;
+
 /*
  * The high 4 bits in ul_info may be used freely by rmgr. The lower 4 bits are
  * not used for now.
@@ -57,6 +65,20 @@ typedef struct xl_ulog_write
 	unsigned char bytes[FLEXIBLE_ARRAY_MEMBER];
 } xl_ulog_write;
 
+extern Size UndoLogShmemSize(void);
+extern void UndoLogShmemInit(void);
+extern void InitUndoLog(void);
+extern void UndoLogWrite(RmgrId rmgr, uint8 info, void *data, int len);
+extern void AtEOXact_UndoLog(bool isCommit);
+extern void AtEOSubXact_UndoLog(bool isCommit);
+extern void AtPrepare_UndoLog(void);
+extern void UndoLog_UndoByXid(bool isCommit, TransactionId xid,
+							  int nchildren, TransactionId *children,
+							  bool redo);
+extern void UndoLogCleanup(bool recovery_end);
+extern void UndoLogRecoveryEnd(void);
+extern void CheckPointUndoLog(void);
+
 extern void undolog_redo(XLogReaderState *record);
 extern void undolog_desc(StringInfo buf, XLogReaderState *record);
 extern const char *undolog_identify(uint8 info);
diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h
index d70e6d37e09..d3cbfefda60 100644
--- a/src/include/storage/lwlock.h
+++ b/src/include/storage/lwlock.h
@@ -215,6 +215,9 @@ typedef enum BuiltinTrancheIds
 	LWTRANCHE_SUBTRANS_SLRU,
 	LWTRANCHE_XACT_SLRU,
 	LWTRANCHE_PARALLEL_VACUUM_DSA,
+	LWTRANCHE_UNDOLOG_DSA,
+	LWTRANCHE_UNDOLOG_HASH,
+	LWTRANCHE_UNDOLOG_DATA,
 	LWTRANCHE_FIRST_USER_DEFINED,
 }			BuiltinTrancheIds;
 
diff --git a/src/include/storage/lwlocklist.h b/src/include/storage/lwlocklist.h
index 88dc79b2bd6..cdee38e791a 100644
--- a/src/include/storage/lwlocklist.h
+++ b/src/include/storage/lwlocklist.h
@@ -84,3 +84,4 @@ PG_LWLOCK(50, DSMRegistry)
 PG_LWLOCK(51, InjectionPoint)
 PG_LWLOCK(52, SerialControl)
 PG_LWLOCK(53, WaitLSN)
+PG_LWLOCK(54, UndoLog)
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index 27e7bfde48f..edca81d88b0 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -2477,6 +2477,7 @@ RewriteState
 RmgrData
 RmgrDescData
 RmgrId
+RmgrUndoData
 RoleNameEntry
 RoleNameItem
 RoleSpec
@@ -3030,10 +3031,15 @@ UINT
 ULARGE_INTEGER
 ULONG
 ULONG_PTR
+ULogOp
+ULogStateData
 UV
 UVersionInfo
 UndoDescData
+UndoLogCtrlStruct
+UndoLogEntry
 UndoLogFileHeader
+UndoLogHashEntry
 UndoLogRecord
 UnicodeNormalizationForm
 UnicodeNormalizationQC
-- 
2.43.5

