From c5a2fb90d44d5fed4c9c8c22f03a9d276b94ba4b Mon Sep 17 00:00:00 2001 From: Rishu Bagga Date: Thu, 15 Sep 2022 09:57:58 +0000 Subject: [PATCH 4/4] slru_to_buf_page_headers_with_upgrade --- src/backend/access/transam/clog.c | 46 ++++--- src/backend/access/transam/commit_ts.c | 60 +++++++-- src/backend/access/transam/multixact.c | 76 ++++++----- src/backend/access/transam/subtrans.c | 12 +- src/backend/commands/async.c | 40 +++--- src/backend/storage/buffer/buf_init.c | 8 -- src/backend/storage/buffer/bufmgr.c | 27 +--- src/backend/storage/lmgr/predicate.c | 10 +- src/backend/storage/page/bufpage.c | 23 +++- src/bin/pg_upgrade/file.c | 175 ++++++++++++++++++++++++- src/bin/pg_upgrade/function.c | 66 ++++++++++ src/bin/pg_upgrade/pg_upgrade.c | 74 +++++++++-- src/bin/pg_upgrade/pg_upgrade.h | 19 ++- src/include/access/nreldefs.h | 19 +++ src/include/access/slru.h | 2 + src/include/storage/bufmgr.h | 5 +- src/include/storage/bufpage.h | 17 +++ 17 files changed, 556 insertions(+), 123 deletions(-) create mode 100644 src/include/access/nreldefs.h diff --git a/src/backend/access/transam/clog.c b/src/backend/access/transam/clog.c index 88a275f5823..67920274171 100644 --- a/src/backend/access/transam/clog.c +++ b/src/backend/access/transam/clog.c @@ -63,7 +63,7 @@ /* We need two bits per xact, so four xacts fit in a byte */ #define CLOG_BITS_PER_XACT 2 #define CLOG_XACTS_PER_BYTE 4 -#define CLOG_XACTS_PER_PAGE (BLCKSZ * CLOG_XACTS_PER_BYTE) +#define CLOG_XACTS_PER_PAGE ((BLCKSZ - SizeOfPageHeaderData) * CLOG_XACTS_PER_BYTE) #define CLOG_XACT_BITMASK ((1 << CLOG_BITS_PER_XACT) - 1) #define TransactionIdToPage(xid) ((xid) / (TransactionId) CLOG_XACTS_PER_PAGE) @@ -88,7 +88,7 @@ static Buffer ZeroCLOGPage(int pageno, bool writeXlog); static bool CLOGPagePrecedes(int page1, int page2); -static void WriteZeroPageXlogRec(int pageno); +static XLogRecPtr WriteZeroPageXlogRec(int pageno); static void WriteTruncateXlogRec(int pageno, TransactionId oldestXact, Oid oldestXactDb); static void TransactionIdSetPageStatus(TransactionId xid, int nsubxids, @@ -353,6 +353,7 @@ TransactionIdSetPageStatusInternal(TransactionId xid, int nsubxids, * we think. */ buffer = ReadNrelBuffer(NREL_CLOG_REL_ID, pageno); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); /* @@ -389,6 +390,7 @@ TransactionIdSetPageStatusInternal(TransactionId xid, int nsubxids, TransactionIdSetStatusBit(subxids[i], status, lsn, buffer); } + MarkBufferDirty(buffer); UnlockReleaseBuffer(buffer); } @@ -575,7 +577,7 @@ TransactionIdSetStatusBit(TransactionId xid, XidStatus status, XLogRecPtr lsn, B Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(GetBufferDescriptor(buffer - 1)), LW_EXCLUSIVE)); - byteptr = BufferGetPage(buffer) + byteno; + byteptr = PageGetContents(BufferGetPage(buffer)) + byteno; curval = (*byteptr >> bshift) & CLOG_XACT_BITMASK; /* @@ -603,6 +605,7 @@ TransactionIdSetStatusBit(TransactionId xid, XidStatus status, XLogRecPtr lsn, B byteval |= (status << bshift); *byteptr = byteval; + /* * Update the buffer LSN if the transaction completion LSN is higher. * @@ -613,8 +616,8 @@ TransactionIdSetStatusBit(TransactionId xid, XidStatus status, XLogRecPtr lsn, B */ if (!XLogRecPtrIsInvalid(lsn)) { - if (BufferGetExternalLSN(GetBufferDescriptor(buffer)) < lsn) - BufferSetExternalLSN(GetBufferDescriptor(buffer), lsn); + if (PageGetLSN(BufferGetPage(buffer)) < lsn) + PageSetLSN(BufferGetPage(buffer), lsn); } } @@ -644,11 +647,11 @@ TransactionIdGetStatus(TransactionId xid, XLogRecPtr *lsn) Buffer buffer; buffer = ReadNrelBuffer(NREL_CLOG_REL_ID, pageno); - byteptr = BufferGetPage(buffer) + byteno; - status = (*byteptr >> bshift) & CLOG_XACT_BITMASK; + byteptr = PageGetContents(BufferGetPage(buffer)) + byteno; - *lsn = BufferGetExternalLSN(GetBufferDescriptor(buffer)); + status = (*byteptr >> bshift) & CLOG_XACT_BITMASK; + *lsn = PageGetLSN(BufferGetPage(buffer)); ReleaseBuffer(buffer); @@ -688,13 +691,20 @@ static Buffer ZeroCLOGPage(int pageno, bool writeXlog) { Buffer buffer; + Page page; + XLogRecPtr lsn; buffer = ZeroNrelBuffer(NREL_CLOG_REL_ID, pageno); + + page = BufferGetPage(buffer); + PageInitNREL(page, BLCKSZ, 0); - MarkBufferDirty(buffer); - + lsn = 0; if (writeXlog) - WriteZeroPageXlogRec(pageno); + lsn = WriteZeroPageXlogRec(pageno); + + PageSetHeaderDataNonRel(page, pageno, lsn, BLCKSZ, PG_METAPAGE_LAYOUT_VERSION); + MarkBufferDirty(buffer); return buffer; } @@ -738,13 +748,16 @@ TrimCLOG(void) buffer = ReadNrelBuffer(NREL_CLOG_REL_ID, pageno); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); - byteptr = BufferGetPage(buffer) + byteno; + byteptr = PageGetContents(BufferGetPage(buffer)) + byteno; /* Zero so-far-unused positions in the current byte */ *byteptr &= (1 << bshift) - 1; /* Zero the rest of the page */ - MemSet(byteptr + 1, 0, BLCKSZ - byteno - 1); + MemSet(byteptr + 1, 0, BLCKSZ - byteno - MAXALIGN(SizeOfPageHeaderData) - 1); + + MarkBufferDirty(buffer); @@ -871,12 +884,15 @@ CLOGPagePrecedes(int page1, int page2) /* * Write a ZEROPAGE xlog record */ -static void +static XLogRecPtr WriteZeroPageXlogRec(int pageno) { + XLogRecPtr lsn; XLogBeginInsert(); XLogRegisterData((char *) (&pageno), sizeof(int)); - (void) XLogInsert(RM_CLOG_ID, CLOG_ZEROPAGE); + lsn = XLogInsert(RM_CLOG_ID, CLOG_ZEROPAGE); + + return lsn; } /* diff --git a/src/backend/access/transam/commit_ts.c b/src/backend/access/transam/commit_ts.c index 6cb2870e5f4..c522c21d6cd 100644 --- a/src/backend/access/transam/commit_ts.c +++ b/src/backend/access/transam/commit_ts.c @@ -64,13 +64,15 @@ typedef struct CommitTimestampEntry sizeof(RepOriginId)) #define COMMIT_TS_XACTS_PER_PAGE \ - (BLCKSZ / SizeOfCommitTimestampEntry) + ((BLCKSZ - SizeOfPageHeaderData) / SizeOfCommitTimestampEntry) #define TransactionIdToCTsPage(xid) \ ((xid) / (TransactionId) COMMIT_TS_XACTS_PER_PAGE) #define TransactionIdToCTsEntry(xid) \ ((xid) % (TransactionId) COMMIT_TS_XACTS_PER_PAGE) + + /* * We keep a cache of the last value set in shared memory. * @@ -105,7 +107,8 @@ static Buffer ZeroCommitTsPage(int pageno, bool writeXlog); static bool CommitTsPagePrecedes(int page1, int page2); static void ActivateCommitTs(void); static void DeactivateCommitTs(void); -static void WriteZeroPageXlogRec(int pageno); +static XLogRecPtr WriteZeroPageXlogRec(int pageno); + static void WriteTruncateXlogRec(int pageno, TransactionId oldestXid); /* @@ -215,6 +218,7 @@ SetXidCommitTsInPage(TransactionId xid, int nsubxids, buffer = ReadNrelBuffer(NREL_COMMITTS_REL_ID, pageno); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); TransactionIdSetCommitTs(xid, ts, nodeid, buffer); @@ -234,6 +238,8 @@ TransactionIdSetCommitTs(TransactionId xid, TimestampTz ts, RepOriginId nodeid, Buffer buffer) { int entryno = TransactionIdToCTsEntry(xid); + int pageno = TransactionIdToCTsPage(xid); + CommitTimestampEntry entry; Assert(TransactionIdIsNormal(xid)); @@ -241,8 +247,12 @@ TransactionIdSetCommitTs(TransactionId xid, TimestampTz ts, entry.time = ts; entry.nodeid = nodeid; - memcpy(BufferGetPage(buffer) + SizeOfCommitTimestampEntry * entryno, + Assert(xid == pageno * COMMIT_TS_XACTS_PER_PAGE + entryno); + + memcpy(PageGetContents(BufferGetPage(buffer)) + \ + SizeOfCommitTimestampEntry * entryno, &entry, SizeOfCommitTimestampEntry); + } /* @@ -318,12 +328,13 @@ TransactionIdGetCommitTsData(TransactionId xid, TimestampTz *ts, buffer = ReadNrelBuffer(NREL_COMMITTS_REL_ID, pageno); + LockBuffer(buffer, BUFFER_LOCK_SHARE); memcpy(&entry, - BufferGetPage(buffer) + - SizeOfCommitTimestampEntry * entryno, - SizeOfCommitTimestampEntry); + PageGetContents(BufferGetPage(buffer)) + \ + SizeOfCommitTimestampEntry * entryno, + SizeOfCommitTimestampEntry); *ts = entry.time; if (nodeid) @@ -546,14 +557,22 @@ static Buffer ZeroCommitTsPage(int pageno, bool writeXlog) { Buffer buffer; + Page page; + XLogRecPtr lsn; buffer = ZeroNrelBuffer(NREL_COMMITTS_REL_ID, pageno); - - MarkBufferDirty(buffer); - + + page = BufferGetPage(buffer); + PageInitNREL(page, BLCKSZ, 0); + if (writeXlog) - WriteZeroPageXlogRec(pageno); - + { + lsn = WriteZeroPageXlogRec(pageno); + page = BufferGetPage(buffer); + PageSetHeaderDataNonRel(page, pageno, lsn, BLCKSZ, PG_METAPAGE_LAYOUT_VERSION); + } + + MarkBufferDirty(buffer); return buffer; } @@ -677,8 +696,13 @@ ActivateCommitTs(void) if (!NonRelDoesPhysicalPageExist(NREL_COMMITTS_REL_ID, pageno)) { Buffer buffer; + Page page; + buffer = ZeroNrelBuffer(NREL_COMMITTS_REL_ID, pageno); + + page = BufferGetPage(buffer); + PageInitNREL(page, BLCKSZ, 0); MarkBufferDirty(buffer); FlushOneBuffer(buffer); @@ -889,12 +913,16 @@ CommitTsPagePrecedes(int page1, int page2) /* * Write a ZEROPAGE xlog record */ -static void +static XLogRecPtr WriteZeroPageXlogRec(int pageno) { + XLogRecPtr lsn; + XLogBeginInsert(); XLogRegisterData((char *) (&pageno), sizeof(int)); - (void) XLogInsert(RM_COMMIT_TS_ID, COMMIT_TS_ZEROPAGE); + lsn = XLogInsert(RM_COMMIT_TS_ID, COMMIT_TS_ZEROPAGE); + + return lsn; } /* @@ -928,11 +956,15 @@ commit_ts_redo(XLogReaderState *record) { int pageno; Buffer buffer; + Page page; memcpy(&pageno, XLogRecGetData(record), sizeof(int)); buffer = ZeroNrelBuffer(NREL_COMMITTS_REL_ID, pageno); - + + page = BufferGetPage(buffer); + PageInitNREL(page, BLCKSZ, 0); + MarkBufferDirty(buffer); FlushOneBuffer(buffer); UnlockReleaseBuffer(buffer); diff --git a/src/backend/access/transam/multixact.c b/src/backend/access/transam/multixact.c index 810aec99a49..5b87faa8e42 100644 --- a/src/backend/access/transam/multixact.c +++ b/src/backend/access/transam/multixact.c @@ -107,7 +107,7 @@ */ /* We need four bytes per offset */ -#define MULTIXACT_OFFSETS_PER_PAGE (BLCKSZ / sizeof(MultiXactOffset)) +#define MULTIXACT_OFFSETS_PER_PAGE ((BLCKSZ - SizeOfPageHeaderData)/ sizeof(MultiXactOffset)) #define MultiXactIdToOffsetPage(xid) \ ((xid) / (MultiXactOffset) MULTIXACT_OFFSETS_PER_PAGE) @@ -139,7 +139,7 @@ /* size in bytes of a complete group */ #define MULTIXACT_MEMBERGROUP_SIZE \ (sizeof(TransactionId) * MULTIXACT_MEMBERS_PER_MEMBERGROUP + MULTIXACT_FLAGBYTES_PER_GROUP) -#define MULTIXACT_MEMBERGROUPS_PER_PAGE (BLCKSZ / MULTIXACT_MEMBERGROUP_SIZE) +#define MULTIXACT_MEMBERGROUPS_PER_PAGE ((BLCKSZ - SizeOfPageHeaderData) / MULTIXACT_MEMBERGROUP_SIZE) #define MULTIXACT_MEMBERS_PER_PAGE \ (MULTIXACT_MEMBERGROUPS_PER_PAGE * MULTIXACT_MEMBERS_PER_MEMBERGROUP) @@ -162,9 +162,9 @@ /* Location (byte offset within page) of flag word for a given member */ #define MXOffsetToFlagsOffset(xid) \ - ((((xid) / (TransactionId) MULTIXACT_MEMBERS_PER_MEMBERGROUP) % \ + (((((xid) / (TransactionId) MULTIXACT_MEMBERS_PER_MEMBERGROUP) % \ (TransactionId) MULTIXACT_MEMBERGROUPS_PER_PAGE) * \ - (TransactionId) MULTIXACT_MEMBERGROUP_SIZE) + (TransactionId) MULTIXACT_MEMBERGROUP_SIZE)) #define MXOffsetToFlagsBitShift(xid) \ (((xid) % (TransactionId) MULTIXACT_MEMBERS_PER_MEMBERGROUP) * \ MXACT_MEMBER_BITS_PER_XACT) @@ -357,7 +357,7 @@ static bool MultiXactOffsetWouldWrap(MultiXactOffset boundary, MultiXactOffset start, uint32 distance); static bool SetOffsetVacuumLimit(bool is_startup); static bool find_multixact_start(MultiXactId multi, MultiXactOffset *result); -static void WriteMZeroPageXlogRec(int pageno, uint8 info); +static XLogRecPtr WriteMZeroPageXlogRec(int pageno, uint8 info); static void WriteMTruncateXlogRec(Oid oldestMultiDB, MultiXactId startTruncOff, MultiXactId endTruncOff, @@ -876,10 +876,9 @@ RecordNewMultiXact(MultiXactId multi, MultiXactOffset offset, else offset_buf = ReadNrelBuffer(NREL_MULTIXACT_OFFSET_REL_ID, pageno); - LockBuffer(offset_buf, BUFFER_LOCK_EXCLUSIVE); - offptr = (MultiXactOffset *) BufferGetPage(offset_buf); + offptr = (MultiXactOffset *) PageGetContents(BufferGetPage(offset_buf)); offptr += entryno; *offptr = offset; @@ -921,16 +920,15 @@ RecordNewMultiXact(MultiXactId multi, MultiXactOffset offset, else buffer = ReadNrelBuffer(NREL_MULTIXACT_MEMBER_REL_ID, pageno); - LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); prev_pageno = pageno; } - memberptr = (TransactionId *) (BufferGetPage(buffer) + memberoff); + memberptr = (TransactionId *) (PageGetContents(BufferGetPage(buffer)) + memberoff); *memberptr = members[i].xid; - flagsptr = (uint32 *) (BufferGetPage(buffer) + flagsoff); + flagsptr = (uint32 *) (PageGetContents(BufferGetPage(buffer)) + flagsoff); flagsval = *flagsptr; flagsval &= ~(((1 << MXACT_MEMBER_BITS_PER_XACT) - 1) << bshift); @@ -1369,7 +1367,7 @@ retry: buffer = ReadNrelBuffer(NREL_MULTIXACT_OFFSET_REL_ID, pageno); LockBuffer(buffer, BUFFER_LOCK_SHARE); - offptr = (MultiXactOffset *) BufferGetPage(buffer); + offptr = (MultiXactOffset *) (PageGetContents(BufferGetPage(buffer))); offptr += entryno; offset = *offptr; @@ -1404,10 +1402,11 @@ retry: UnlockReleaseBuffer(buffer); buffer = ReadNrelBuffer(NREL_MULTIXACT_OFFSET_REL_ID, pageno); + LockBuffer(buffer, BUFFER_LOCK_SHARE); } - offptr = (MultiXactOffset *) BufferGetPage(buffer); + offptr = (MultiXactOffset *) (PageGetContents(BufferGetPage(buffer))); offptr += entryno; nextMXOffset = *offptr; @@ -1450,7 +1449,7 @@ retry: prev_pageno = pageno; } - xactptr = (TransactionId *) (BufferGetPage(buffer) + memberoff); + xactptr = (TransactionId *) (PageGetContents((BufferGetPage(buffer)) + memberoff)); if (!TransactionIdIsValid(*xactptr)) { @@ -1461,7 +1460,7 @@ retry: flagsoff = MXOffsetToFlagsOffset(offset); bshift = MXOffsetToFlagsBitShift(offset); - flagsptr = (uint32 *) (BufferGetPage(buffer) + flagsoff); + flagsptr = (uint32 *) (PageGetContents(BufferGetPage(buffer)) + flagsoff); ptr[truelength].xid = *xactptr; ptr[truelength].status = (*flagsptr >> bshift) & MXACT_MEMBER_XACT_BITMASK; @@ -1926,13 +1925,21 @@ static Buffer ZeroMultiXactOffsetPage(int pageno, bool writeXlog) { Buffer buffer; + Page page; + XLogRecPtr recptr; - buffer = ZeroNrelBuffer(NREL_MULTIXACT_OFFSET_REL_ID, pageno); - MarkBufferDirty(buffer); + buffer = ZeroNrelBuffer(NREL_MULTIXACT_OFFSET_REL_ID, pageno); + page = BufferGetPage(buffer); + PageInitNREL(page, BLCKSZ, 0); + recptr = 0; + if (writeXlog) - WriteMZeroPageXlogRec(pageno, XLOG_MULTIXACT_ZERO_OFF_PAGE); + recptr = WriteMZeroPageXlogRec(pageno, XLOG_MULTIXACT_ZERO_OFF_PAGE); + + PageSetHeaderDataNonRel(page, pageno, recptr, BLCKSZ, PG_METAPAGE_LAYOUT_VERSION); + MarkBufferDirty(buffer); return buffer; } @@ -1944,14 +1951,20 @@ static Buffer ZeroMultiXactMemberPage(int pageno, bool writeXlog) { Buffer buffer; + Page page; + XLogRecPtr recptr; buffer = ZeroNrelBuffer(NREL_MULTIXACT_MEMBER_REL_ID, pageno); - - MarkBufferDirty(buffer); - + page = BufferGetPage(buffer); + PageInitNREL(page, BLCKSZ, 0); + + recptr = 0; if (writeXlog) - WriteMZeroPageXlogRec(pageno, XLOG_MULTIXACT_ZERO_MEM_PAGE); - + recptr = WriteMZeroPageXlogRec(pageno, XLOG_MULTIXACT_ZERO_MEM_PAGE); + + PageSetHeaderDataNonRel(page, pageno, recptr, BLCKSZ, PG_METAPAGE_LAYOUT_VERSION); + MarkBufferDirty(buffer); + return buffer; } @@ -2041,10 +2054,10 @@ TrimMultiXact(void) buffer = ReadNrelBuffer(NREL_MULTIXACT_OFFSET_REL_ID, pageno); LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); - offptr = (MultiXactOffset *) BufferGetPage(buffer); + offptr = (MultiXactOffset *) (PageGetContents(BufferGetPage(buffer))); offptr += entryno; - MemSet(offptr, 0, BLCKSZ - (entryno * sizeof(MultiXactOffset))); + MemSet(offptr, 0, BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - (entryno * sizeof(MultiXactOffset))); MarkBufferDirty(buffer); UnlockReleaseBuffer(buffer); @@ -2067,9 +2080,9 @@ TrimMultiXact(void) buffer = ReadNrelBuffer(NREL_MULTIXACT_MEMBER_REL_ID, pageno); LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); - xidptr = (TransactionId *) (BufferGetPage(buffer) + memberoff); + xidptr = (TransactionId *) (PageGetContents(BufferGetPage(buffer)) + memberoff); - MemSet(xidptr, 0, BLCKSZ - memberoff); + MemSet(xidptr, 0, BLCKSZ - memberoff - MAXALIGN(SizeOfPageHeaderData)); /* * Note: we don't need to zero out the flag bits in the remaining @@ -2419,7 +2432,6 @@ ExtendMultiXactMember(MultiXactOffset offset, int nmembers, Buffer ** buffers) { /* do a read buffer call to allocate space beforehand */ buf = ReadNrelBuffer(NREL_MULTIXACT_MEMBER_REL_ID, pageno); - } if (buffers) @@ -2718,7 +2730,7 @@ find_multixact_start(MultiXactId multi, MultiXactOffset *result) buffer = ReadNrelBuffer(NREL_MULTIXACT_OFFSET_REL_ID, pageno); LockBuffer(buffer, BUFFER_LOCK_SHARE); - offptr = (MultiXactOffset *) BufferGetPage(buffer); + offptr = (MultiXactOffset *) (PageGetContents(BufferGetPage(buffer))); offptr += entryno; offset = *offptr; UnlockReleaseBuffer(buffer); @@ -3138,12 +3150,16 @@ MultiXactOffsetPrecedes(MultiXactOffset offset1, MultiXactOffset offset2) * Write an xlog record reflecting the zeroing of either a MEMBERs or * OFFSETs page (info shows which) */ -static void +static XLogRecPtr WriteMZeroPageXlogRec(int pageno, uint8 info) { + XLogRecPtr recptr; + XLogBeginInsert(); XLogRegisterData((char *) (&pageno), sizeof(int)); - (void) XLogInsert(RM_MULTIXACT_ID, info); + recptr = XLogInsert(RM_MULTIXACT_ID, info); + + return recptr; } /* diff --git a/src/backend/access/transam/subtrans.c b/src/backend/access/transam/subtrans.c index 635e5368cd6..975fc47814e 100644 --- a/src/backend/access/transam/subtrans.c +++ b/src/backend/access/transam/subtrans.c @@ -51,7 +51,7 @@ */ /* We need four bytes per xact */ -#define SUBTRANS_XACTS_PER_PAGE (BLCKSZ / sizeof(TransactionId)) +#define SUBTRANS_XACTS_PER_PAGE ((BLCKSZ - SizeOfPageHeaderData) / sizeof(TransactionId)) #define TransactionIdToPage(xid) ((xid) / (TransactionId) SUBTRANS_XACTS_PER_PAGE) #define TransactionIdToEntry(xid) ((xid) % (TransactionId) SUBTRANS_XACTS_PER_PAGE) @@ -77,8 +77,9 @@ SubTransSetParent(TransactionId xid, TransactionId parent) buffer = ReadNrelBuffer(NREL_SUBTRANS_REL_ID, pageno); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); - ptr = (TransactionId *) BufferGetPage(buffer); + ptr = (TransactionId *) PageGetContents(BufferGetPage(buffer)); ptr += entryno; /* @@ -118,7 +119,8 @@ SubTransGetParent(TransactionId xid) buffer = ReadNrelBuffer(NREL_SUBTRANS_REL_ID, pageno); - ptr = (TransactionId *) BufferGetPage(buffer); + + ptr = (TransactionId *) PageGetContents(BufferGetPage(buffer)); ptr += entryno; parent = *ptr; @@ -208,9 +210,13 @@ static Buffer ZeroSUBTRANSPage(int pageno) { Buffer buffer; + Page page; buffer = ZeroNrelBuffer(NREL_SUBTRANS_REL_ID, pageno); + page = BufferGetPage(buffer); + PageInitNREL(page, BLCKSZ, 0); + MarkBufferDirty(buffer); return buffer; diff --git a/src/backend/commands/async.c b/src/backend/commands/async.c index 723cf8f6e1c..6023133c809 100644 --- a/src/backend/commands/async.c +++ b/src/backend/commands/async.c @@ -165,7 +165,7 @@ * than that, so changes in that data structure won't affect user-visible * restrictions. */ -#define NOTIFY_PAYLOAD_MAX_LENGTH (BLCKSZ - NAMEDATALEN - 128) +#define NOTIFY_PAYLOAD_MAX_LENGTH (BLCKSZ - NAMEDATALEN - SizeOfPageHeaderData - 128) /* * Struct representing an entry in the global notify queue @@ -215,7 +215,7 @@ typedef struct QueuePosition ((x).page == (y).page && (x).offset == (y).offset) #define QUEUE_POS_IS_ZERO(x) \ - ((x).page == 0 && (x).offset == 0) + ((x).page == 0 && (x).offset == MAXALIGN(SizeOfPageHeaderData)) /* choose logically smaller QueuePosition */ #define QUEUE_POS_MIN(x,y) \ @@ -544,8 +544,8 @@ AsyncShmemInit(void) if (!found) { /* First time through, so initialize it */ - SET_QUEUE_POS(QUEUE_HEAD, 0, 0); - SET_QUEUE_POS(QUEUE_TAIL, 0, 0); + SET_QUEUE_POS(QUEUE_HEAD, 0, MAXALIGN(SizeOfPageHeaderData)); + SET_QUEUE_POS(QUEUE_TAIL, 0, MAXALIGN(SizeOfPageHeaderData)); QUEUE_STOP_PAGE = 0; QUEUE_FIRST_LISTENER = InvalidBackendId; asyncQueueControl->lastQueueFillWarn = 0; @@ -555,7 +555,7 @@ AsyncShmemInit(void) QUEUE_BACKEND_PID(i) = InvalidPid; QUEUE_BACKEND_DBOID(i) = InvalidOid; QUEUE_NEXT_LISTENER(i) = InvalidBackendId; - SET_QUEUE_POS(QUEUE_BACKEND_POS(i), 0, 0); + SET_QUEUE_POS(QUEUE_BACKEND_POS(i), 0, MAXALIGN(SizeOfPageHeaderData)); } } @@ -1332,19 +1332,19 @@ asyncQueueAdvance(volatile QueuePosition *position, int entryLength) * written or read. */ offset += entryLength; - Assert(offset <= QUEUE_PAGESIZE); + Assert(offset <= QUEUE_PAGESIZE - MAXALIGN(SizeOfPageHeaderData)); /* * In a second step check if another entry can possibly be written to the * page. If so, stay here, we have reached the next position. If not, then * we need to move on to the next page. */ - if (offset + QUEUEALIGN(AsyncQueueEntryEmptySize) > QUEUE_PAGESIZE) + if (offset + QUEUEALIGN(AsyncQueueEntryEmptySize) > QUEUE_PAGESIZE - MAXALIGN(SizeOfPageHeaderData)) { pageno++; if (pageno > QUEUE_MAX_PAGE) pageno = 0; /* wrap around */ - offset = 0; + offset = MAXALIGN(SizeOfPageHeaderData); /* start at SizeOfPageHeaderData */ pageJump = true; } @@ -1399,6 +1399,7 @@ asyncQueueAddEntries(ListCell *nextNotify) int pageno; int offset; Buffer buffer; + Page page; /* * We work with a local copy of QUEUE_HEAD, which we write back to shared @@ -1424,15 +1425,22 @@ asyncQueueAddEntries(ListCell *nextNotify) pageno = QUEUE_POS_PAGE(queue_head); if (QUEUE_POS_IS_ZERO(queue_head)) { + buffer = ZeroNrelBuffer(NREL_NOTIFY_REL_ID, pageno); + page = BufferGetPage(buffer); + PageInitNREL(page, BLCKSZ, 0); + PageSetHeaderDataNonRel(page, pageno, InvalidXLogRecPtr, BLCKSZ, PG_METAPAGE_LAYOUT_VERSION); + } else { buffer = ReadNrelBuffer(NREL_NOTIFY_REL_ID, pageno); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); } + /* Note we mark the page dirty before writing in it */ MarkBufferDirty(buffer); @@ -1446,7 +1454,7 @@ asyncQueueAddEntries(ListCell *nextNotify) offset = QUEUE_POS_OFFSET(queue_head); /* Check whether the entry really fits on the current page */ - if (offset + qe.length <= QUEUE_PAGESIZE) + if (offset + qe.length <= QUEUE_PAGESIZE - MAXALIGN(SizeOfPageHeaderData)) { /* OK, so advance nextNotify past this item */ nextNotify = lnext(pendingNotifies->events, nextNotify); @@ -1458,17 +1466,18 @@ asyncQueueAddEntries(ListCell *nextNotify) * only check dboid and since it won't match any reader's database * OID, they will ignore this entry and move on. */ - qe.length = QUEUE_PAGESIZE - offset; + qe.length = QUEUE_PAGESIZE - MAXALIGN(SizeOfPageHeaderData) - offset; qe.dboid = InvalidOid; qe.data[0] = '\0'; /* empty channel */ qe.data[1] = '\0'; /* empty payload */ } /* Now copy qe into the shared buffer page */ - memcpy(BufferGetPage(buffer) + offset, + memcpy(PageGetContents(BufferGetPage(buffer)) + offset, &qe, qe.length); + /* Advance queue_head appropriately, and detect if page is full */ if (asyncQueueAdvance(&(queue_head), qe.length)) { @@ -1985,6 +1994,7 @@ asyncQueueReadAllNotifications(void) */ buffer = ReadNrelBuffer(NREL_NOTIFY_REL_ID, curpage); + if (curpage == QUEUE_POS_PAGE(head)) { /* we only want to read as far as head */ @@ -1995,10 +2005,10 @@ asyncQueueReadAllNotifications(void) else { /* fetch all the rest of the page */ - copysize = QUEUE_PAGESIZE - curoffset; + copysize = QUEUE_PAGESIZE - MAXALIGN(SizeOfPageHeaderData) - curoffset; } - memcpy(page_buffer.buf + curoffset, - BufferGetPage(buffer) + curoffset, + memcpy(PageGetContents(page_buffer.buf) + curoffset, + PageGetContents(BufferGetPage(buffer)) + curoffset, copysize); ReleaseBuffer(buffer); @@ -2068,7 +2078,7 @@ asyncQueueProcessPageEntries(volatile QueuePosition *current, if (QUEUE_POS_EQUAL(thisentry, stop)) break; - qe = (AsyncQueueEntry *) (page_buffer + QUEUE_POS_OFFSET(thisentry)); + qe = (AsyncQueueEntry *) (PageGetContents(page_buffer) + QUEUE_POS_OFFSET(thisentry)); /* * Advance *current over this message, possibly to the next page. As diff --git a/src/backend/storage/buffer/buf_init.c b/src/backend/storage/buffer/buf_init.c index e587be6498b..559ebdc3338 100644 --- a/src/backend/storage/buffer/buf_init.c +++ b/src/backend/storage/buffer/buf_init.c @@ -20,7 +20,6 @@ BufferDescPadded *BufferDescriptors; char *BufferBlocks; -XLogRecPtr *BufferExternalLSNs; ConditionVariableMinimallyPadded *BufferIOCVArray; WritebackContext BackendWritebackContext; CkptSortItem *CkptBufferIds; @@ -94,11 +93,6 @@ InitBufferPool(void) NBuffers * sizeof(ConditionVariableMinimallyPadded), &foundIOCV); - BufferExternalLSNs = (XLogRecPtr *) - ShmemInitStruct("Buffer External LSNs", - NBuffers * sizeof(XLogRecPtr), - &foundLSNs); - /* * The array used to sort to-be-checkpointed buffer ids is located in * shared memory, to avoid having to allocate significant amounts of @@ -144,8 +138,6 @@ InitBufferPool(void) LWTRANCHE_BUFFER_CONTENT); ConditionVariableInit(BufferDescriptorGetIOCV(buf)); - - BufferExternalLSNs[i] = InvalidXLogRecPtr; } /* Correct last entry of linked list */ diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index c590f05a0d9..7ef7f9b0617 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -63,16 +63,12 @@ * meantime we need some way to identify buffers that hold raw data (no * invasive LSN, no checksums). */ -#define BufferHasStandardPage(bufHdr) \ - ((bufHdr)->tag.dbOid != 9) - -#define BufferHasExternalLSN(bufHdr) \ - !BufferHasStandardPage(bufHdr) /* Note: these two macros only work on shared buffers, not local ones! */ #define BufHdrGetBlock(bufHdr) ((Block) (BufferBlocks + ((Size) (bufHdr)->buf_id) * BLCKSZ)) + #define BufferGetLSN(bufHdr) \ - (BufferHasExternalLSN(bufHdr) ? BufferGetExternalLSN(bufHdr) : PageGetLSN(BufHdrGetBlock(bufHdr))) + PageGetLSN(BufHdrGetBlock(bufHdr)) /* Note: this macro only works on local buffers, not shared ones! */ #define LocalBufHdrGetBlock(bufHdr) \ @@ -1155,8 +1151,7 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, IOOP_READ, io_start, 1); /* check for garbage data */ - if (BufferHasStandardPage(bufHdr) && - (!PageIsVerifiedExtended((Page) bufBlock, blockNum, + if ((!(mode == RBM_TRIM)) && (!PageIsVerifiedExtended((Page) bufBlock, blockNum, PIV_LOG_WARNING | PIV_REPORT_STAT))) { if (mode == RBM_ZERO_ON_ERROR || zero_damaged_pages) @@ -1388,8 +1383,6 @@ BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, return existing_buf_hdr; } - if (BufferHasExternalLSN(victim_buf_hdr)) - BufferSetExternalLSN(victim_buf_hdr, InvalidXLogRecPtr); /* * Need to lock the buffer header too in order to change its tag. @@ -3457,10 +3450,7 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, * buffer, other processes might be updating hint bits in it, so we must * copy the page to private storage if we do checksumming. */ - if (BufferHasStandardPage(buf)) - bufToWrite = PageSetChecksumCopy((Page) bufBlock, buf->tag.blockNum); - else - bufToWrite = bufBlock; + bufToWrite = PageSetChecksumCopy((Page) bufBlock, buf->tag.blockNum); io_start = pgstat_prepare_io_time(); @@ -3601,10 +3591,8 @@ BufferGetLSNAtomic(Buffer buffer) Assert(BufferIsPinned(buffer)); buf_state = LockBufHdr(bufHdr); - if (BufferHasStandardPage(bufHdr)) - lsn = PageGetLSN(page); - else - lsn = BufferGetExternalLSN(bufHdr); + lsn = PageGetLSN(page); + UnlockBufHdr(bufHdr, buf_state); return lsn; @@ -4199,8 +4187,7 @@ FlushRelationBuffers(Relation rel) errcallback.previous = error_context_stack; error_context_stack = &errcallback; - if (BufferHasStandardPage(bufHdr)) - PageSetChecksumInplace(localpage, bufHdr->tag.blockNum); + PageSetChecksumInplace(localpage, bufHdr->tag.blockNum); io_start = pgstat_prepare_io_time(); diff --git a/src/backend/storage/lmgr/predicate.c b/src/backend/storage/lmgr/predicate.c index 2177386ca50..d1591dd9710 100644 --- a/src/backend/storage/lmgr/predicate.c +++ b/src/backend/storage/lmgr/predicate.c @@ -316,7 +316,7 @@ #define SERIAL_PAGESIZE BLCKSZ #define SERIAL_ENTRYSIZE sizeof(SerCommitSeqNo) -#define SERIAL_ENTRIESPERPAGE (SERIAL_PAGESIZE / SERIAL_ENTRYSIZE) +#define SERIAL_ENTRIESPERPAGE ((SERIAL_PAGESIZE - SizeOfPageHeaderData) / SERIAL_ENTRYSIZE) /* * Set maximum pages based on the number needed to track all transactions. @@ -326,11 +326,12 @@ #define SerialNextPage(page) (((page) >= SERIAL_MAX_PAGE) ? 0 : (page) + 1) #define SerialValue(buffer, xid) (*((SerCommitSeqNo *) \ - (BufferGetPage(buffer) + \ + (PageGetContents(BufferGetPage(buffer)) + \ ((((uint32) (xid)) % SERIAL_ENTRIESPERPAGE) * SERIAL_ENTRYSIZE)))) #define SerialPage(xid) (((uint32) (xid)) / SERIAL_ENTRIESPERPAGE) + typedef struct SerialControlData { int headPage; /* newest initialized page */ @@ -781,7 +782,7 @@ SerialPagePrecedesLogicallyUnitTests(void) */ headPage = oldestPage; targetPage = newestPage; - Assert(SerialPagePrecedesLogically(headPage, targetPage - 1)); + Assert(SerialPagePrecedesLogically(headPage, targetPage - 2)); #if 0 Assert(SerialPagePrecedesLogically(headPage, targetPage)); #endif @@ -878,6 +879,7 @@ SerialAdd(TransactionId xid, SerCommitSeqNo minConflictCommitSeqNo) { buffer = ZeroNrelBuffer(NREL_SERIAL_REL_ID, firstZeroPage); + PageSetHeaderDataNonRel(BufferGetPage(buffer), firstZeroPage, InvalidXLogRecPtr, BLCKSZ, PG_METAPAGE_LAYOUT_VERSION); MarkBufferDirty(buffer); UnlockReleaseBuffer(buffer); firstZeroPage = SerialNextPage(firstZeroPage); @@ -888,6 +890,7 @@ SerialAdd(TransactionId xid, SerCommitSeqNo minConflictCommitSeqNo) { buffer = ReadNrelBuffer(NREL_SERIAL_REL_ID, targetPage); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); } @@ -933,6 +936,7 @@ SerialGetMinConflictCommitSeqNo(TransactionId xid) */ buffer = ReadNrelBuffer(NREL_SERIAL_REL_ID, SerialPage(xid)); + val = SerialValue(buffer, xid); ReleaseBuffer(buffer); LWLockRelease(SerialSLRULock); diff --git a/src/backend/storage/page/bufpage.c b/src/backend/storage/page/bufpage.c index 9a302ddc30e..bc2f5f97b00 100644 --- a/src/backend/storage/page/bufpage.c +++ b/src/backend/storage/page/bufpage.c @@ -59,6 +59,27 @@ PageInit(Page page, Size pageSize, Size specialSize) /* p->pd_prune_xid = InvalidTransactionId; done by above MemSet */ } +void +PageInitNREL(Page page, Size pageSize, Size specialSize) +{ + PageHeader p = (PageHeader) page; + + specialSize = MAXALIGN(specialSize); + + Assert(pageSize == BLCKSZ); + Assert(pageSize > specialSize + SizeOfPageHeaderData); + + /* Make sure all fields of page are zero, as well as unused space */ + MemSet(p, 0, pageSize); + + p->pd_flags = 0; + p->pd_lower = SizeOfPageHeaderData; + p->pd_upper = pageSize - specialSize; + p->pd_special = pageSize - specialSize; + PageSetPageSizeAndVersion(page, pageSize, PG_METAPAGE_LAYOUT_VERSION); + /* p->pd_prune_xid = InvalidTransactionId; done by above MemSet */ +} + /* * PageIsVerifiedExtended @@ -103,7 +124,7 @@ PageIsVerifiedExtended(Page page, BlockNumber blkno, int flags) if (DataChecksumsEnabled()) { checksum = pg_checksum_page((char *) page, blkno); - + if (checksum != p->pd_checksum) checksum_failure = true; } diff --git a/src/bin/pg_upgrade/file.c b/src/bin/pg_upgrade/file.c index d1736028826..48dfb8505cc 100644 --- a/src/bin/pg_upgrade/file.c +++ b/src/bin/pg_upgrade/file.c @@ -20,6 +20,7 @@ #endif #include "access/visibilitymapdefs.h" +#include "access/nreldefs.h" #include "common/file_perm.h" #include "pg_upgrade.h" #include "storage/bufpage.h" @@ -139,7 +140,6 @@ copyFile(const char *src, const char *dst, #endif /* WIN32 */ } - /* * linkFile() * @@ -316,6 +316,179 @@ rewriteVisibilityMap(const char *fromfile, const char *tofile, close(src_fd); } + +//create new file and initialize to 256K +static int +init_new_slru_file(FILE * fp) +{ + int fd; + int res; + + fd = fileno(fp); + res = ftruncate(fd, BLCKSZ * SLRU_PAGES_PER_SEGMENT); + return res; +} + +int +copy_to_new_format(const char *old_subdir, const char *new_subdir, int element_size) +{ + DIR *dr; + struct dirent *de; + + size_t read_items; + size_t n_items; + size_t to_read_items; + size_t write_items; + size_t pages_written; + size_t write_offset; + + char * read_file; + char * write_file; + + int res; + int write_file_segno; + int total_read_files; + + FILE * old_fd; + FILE * new_fp; + +#define MAXBUFSIZE (BLCKSZ - SizeOfPageHeaderData) + + char old_path[MAXPGPATH]; + char new_path[MAXPGPATH]; + PGAlignedBlock pg_buf; + BlockNumber blkno; //track aboslute block number + + + struct dirent** all_dirents; + + + snprintf(old_path, sizeof(old_path), "%s/%s", old_cluster.pgdata, old_subdir); + snprintf(new_path, sizeof(new_path), "%s/%s", new_cluster.pgdata, new_subdir); + + /* + * calculate the number of items that can fit + * inside BLCKSZ - SizeOfPageHeaderData + */ + + n_items = (size_t) (MAXBUFSIZE / element_size); + + /* number of items to read in at a time */ + to_read_items = n_items; + + //bufsize = (size_t) (n_items * element_size); + write_file_segno = 0; /* track which segment we are writing into */ + write_offset = (size_t) (SizeOfPageHeaderData); + + dr = opendir((char *)old_path); + if (dr == NULL) + return -1; + + write_file = psprintf("%s/%04X", new_path, write_file_segno); + + new_fp = fopen(write_file, "wb"); + + if (new_fp == NULL) + return -1; + + res = init_new_slru_file(new_fp); + if (res < 0) + return -1; + + if (res < 0) + return -1; + + pages_written = 0; + blkno = 0; /* absolute blkno */ + + all_dirents = get_sorted_hex_files(old_path, &total_read_files); + + write_items = 0; + for (int i = 0; i < total_read_files; i++) + { + de = all_dirents[i]; + + if ((!strcmp((char *) de->d_name, ".")) || !strcmp((char *) de->d_name, "..")) + { + continue; + } + + read_file = psprintf("%s/%s", (char *) old_path, (char *) de->d_name); + + old_fd = fopen(read_file, "rb"); + read_items = 0; + + do + { + memset(pg_buf.data, 0, BLCKSZ); + read_items = fread(pg_buf.data + write_offset, element_size, to_read_items, old_fd); /* how many items we read */ + + if (to_read_items == n_items) /* not finishing up a partial read */ + { + + ((PageHeader) pg_buf.data)->pd_lower = SizeOfPageHeaderData; + ((PageHeader) pg_buf.data)->pd_upper = BLCKSZ; + ((PageHeader) pg_buf.data)->pd_special = BLCKSZ; + + ((PageHeader) pg_buf.data)->pd_checksum = + pg_checksum_page(pg_buf.data, blkno); + + fwrite(pg_buf.data, SizeOfPageHeaderData, 1, new_fp); /* write page header data */ + } + + + write_items += fwrite(pg_buf.data + write_offset, element_size, read_items, new_fp); /* increment how many items written */ + + if (write_items % n_items == 0 && write_items > 0 && errno == 0) /* finished writing into new page. */ + { + if (write_items == n_items * SLRU_PAGES_PER_SEGMENT) /* end of segment */ + { + fclose(new_fp); + pages_written = 0; + write_file_segno ++; + write_items = 0; + write_file = psprintf("%s/%04X", new_path, write_file_segno); + new_fp = fopen((char *)write_file, "wb"); + res = init_new_slru_file(new_fp); + if (res < 0 || new_fp == NULL) + { + fclose(new_fp); + cleanup_dirents(all_dirents, total_read_files); + + } + } else { /* end of page but not segment */ + //fflush(new_fp); + pages_written ++; + } + + blkno++; + + to_read_items = n_items; /* to_read_items = n */ + res = fseek(new_fp, (pages_written * BLCKSZ), SEEK_SET); + if (res < 0) + { + fclose(new_fp); + cleanup_dirents(all_dirents, total_read_files); + return -1; + } + } else { + /*end of segment in source dir*/ + if (read_items < to_read_items) + { + to_read_items = n_items - read_items; /* read remaining items */ + } + } + } while (!feof(old_fd)); /* until end of file */ + fclose(old_fd); + } + fflush(new_fp); + + /* free memory malloc'd by scandir while sorting */ + cleanup_dirents(all_dirents, total_read_files); + + return 0; +} + void check_file_clone(void) { diff --git a/src/bin/pg_upgrade/function.c b/src/bin/pg_upgrade/function.c index dc8800c7cde..10c7fcafbd5 100644 --- a/src/bin/pg_upgrade/function.c +++ b/src/bin/pg_upgrade/function.c @@ -7,6 +7,7 @@ * src/bin/pg_upgrade/function.c */ + #include "postgres_fe.h" #include "access/transam.h" @@ -42,6 +43,43 @@ library_name_compare(const void *p1, const void *p2) ((const LibraryInfo *) p2)->dbnum; } +/* + * qsort comparator for hex filenames + */ +static int +file_name_compare(const struct dirent ** de_1, const struct dirent ** de_2) +{ + int n1; + int n2; + + + char * fname_1; + char * fname_2; + + fname_1 = (char *) (*de_1)->d_name; + fname_2 = (char *) (*de_2)->d_name; + + if ((strcmp(fname_1, ".")) || strcmp(fname_1, "..")) + { + n1 = (int) strtol(fname_1, NULL, 16); + } else { + n1 = -1; + } + + if ((strcmp(fname_2, ".")) || strcmp(fname_2, "..")) + { + n2 = (int) strtol(fname_2, NULL, 16); + } else { + n2 = -1; + } + + if (n1 == n2) + { + return 1; //arbitrarily select the first input + } else { + return n1 - n2; + } +} /* * get_loadable_libraries() @@ -109,6 +147,34 @@ get_loadable_libraries(void) } +/* + * get_sorted_hex_files() + * given the filepath of a directory, + * return array of child dirents with hex filenames e.g '000A' + * in sorted order + */ +struct dirent** +get_sorted_hex_files(char * dr, int * size) +{ + struct dirent **entry_list; + + *size = scandir(dr, &entry_list, NULL, file_name_compare); + if (*size < 0) + { + return NULL; //error + } + return entry_list; +} + +void +cleanup_dirents(struct dirent ** all_dirents, int total_read_files) +{ + for (int i = 0; i < total_read_files; i++) + { + free((struct dirent **) all_dirents[i]); + } + free((struct dirent **) all_dirents); +} /* * check_loadable_libraries() * diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c index 4562dafcff5..4925e4505ac 100644 --- a/src/bin/pg_upgrade/pg_upgrade.c +++ b/src/bin/pg_upgrade/pg_upgrade.c @@ -38,18 +38,22 @@ #include "postgres_fe.h" +#include #include +#include #ifdef HAVE_LANGINFO_H #include #endif +#include "access/nreldefs.h" #include "catalog/pg_class_d.h" #include "common/file_perm.h" #include "common/logging.h" #include "common/restricted_token.h" #include "fe_utils/string_utils.h" #include "pg_upgrade.h" +#include "storage/bufpage.h" static void set_locale_and_encoding(void); static void prepare_new_cluster(void); @@ -60,6 +64,10 @@ static void set_frozenxids(bool minmxid_only); static void make_outputdirs(char *pgdata); static void setup(char *argv0, bool *live_check); + +#define MAXBUFSIZE (BLCKSZ - SizeOfPageHeaderData) +#define SLRU_PAGES_PER_SEGMENT 32 + ClusterInfo old_cluster, new_cluster; OSInfo os_info; @@ -646,11 +654,36 @@ copy_xact_xlog_xid(void) * Copy old commit logs to new data dir. pg_clog has been renamed to * pg_xact in post-10 clusters. */ - copy_subdir_files(GET_MAJOR_VERSION(old_cluster.major_version) <= 906 ? - "pg_clog" : "pg_xact", - GET_MAJOR_VERSION(new_cluster.major_version) <= 906 ? - "pg_clog" : "pg_xact"); + + if (old_cluster.controldata.cat_ver <= CLOG_FORMATCHANGE_CAT_VER && + new_cluster.controldata.cat_ver >= CLOG_FORMATCHANGE_CAT_VER) + { + int ret; + ret = copy_to_new_format(GET_MAJOR_VERSION(old_cluster.major_version) <= 906 ? + "pg_clog" : "pg_xact", + "pg_xact", 1); + + if (ret < 0) + { + pg_fatal("could not reformat clog files"); + } + } + if (old_cluster.controldata.cat_ver >= CLOG_FORMATCHANGE_CAT_VER && + new_cluster.controldata.cat_ver >= CLOG_FORMATCHANGE_CAT_VER) + { + copy_subdir_files("pg_xact", "pg_xact"); + } + + if (old_cluster.controldata.cat_ver <= CLOG_FORMATCHANGE_CAT_VER && + new_cluster.controldata.cat_ver <= CLOG_FORMATCHANGE_CAT_VER) + { + copy_subdir_files(GET_MAJOR_VERSION(old_cluster.major_version) <= 906 ? + "pg_clog" : "pg_xact", + GET_MAJOR_VERSION(new_cluster.major_version) <= 906 ? + "pg_clog" : "pg_xact"); + } + prep_status("Setting oldest XID for new cluster"); exec_prog(UTILITY_LOG_FILE, NULL, true, true, "\"%s/pg_resetwal\" -f -u %u \"%s\"", @@ -706,23 +739,44 @@ copy_xact_xlog_xid(void) } else if (new_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER) { + copy_to_new_format("pg_multixact/offsets", "pg_multixact/offsets", MULTIXACT_MEMBER_ENTRY_SIZE); + copy_to_new_format("pg_multixact/members", "pg_multixact/members", MULTIXACT_OFFSET_ENTRY_SIZE); + + + prep_status("Setting next multixact ID and offset for new cluster"); + /* + * we preserve all files and contents, so we must preserve both "next" + * counters here and the oldest multi present on system. + */ + exec_prog(UTILITY_LOG_FILE, NULL, true, true, + "\"%s/pg_resetwal\" -O %u -m %u,%u \"%s\"", + new_cluster.bindir, + old_cluster.controldata.chkpnt_nxtmxoff, + old_cluster.controldata.chkpnt_nxtmulti, + old_cluster.controldata.chkpnt_oldstMulti, + new_cluster.pgdata); + check_ok(); + + } + /*else if (new_cluster.controldata.cat_ver >= MULTIXACT_FORMATCHANGE_CAT_VER) + { + * * Remove offsets/0000 file created by initdb that no longer matches * the new multi-xid value. "members" starts at zero so no need to * remove it. - */ + * remove_new_subdir("pg_multixact/offsets", false); prep_status("Setting oldest multixact ID in new cluster"); - - /* + * * We don't preserve files in this case, but it's important that the * oldest multi is set to the latest value used by the old system, so * that multixact.c returns the empty set for multis that might be * present on disk. We set next multi to the value following that; it * might end up wrapped around (i.e. 0) if the old cluster had * next=MaxMultiXactId, but multixact.c can cope with that just fine. - */ + * exec_prog(UTILITY_LOG_FILE, NULL, true, true, "\"%s/pg_resetwal\" -m %u,%u \"%s\"", new_cluster.bindir, @@ -730,7 +784,8 @@ copy_xact_xlog_xid(void) old_cluster.controldata.chkpnt_nxtmulti, new_cluster.pgdata); check_ok(); - } + + }*/ /* now reset the wal archives in the new cluster */ prep_status("Resetting WAL archives"); @@ -742,7 +797,6 @@ copy_xact_xlog_xid(void) check_ok(); } - /* * set_frozenxids() * diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 3eea0139c74..7995d1e9635 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -113,7 +114,16 @@ extern char *output_files[]; * version to this value. pg_upgrade behavior depends on whether old and new * server versions are both newer than this, or only the new one is. */ -#define MULTIXACT_FORMATCHANGE_CAT_VER 201301231 + +/* + * page header format change + */ +#define MULTIXACT_FORMATCHANGE_CAT_VER 202209141 + +/* + * page header format change + */ +#define CLOG_FORMATCHANGE_CAT_VER 202210141 /* * large object chunk size added to pg_controldata, @@ -122,7 +132,7 @@ extern char *output_files[]; #define LARGE_OBJECT_SIZE_PG_CONTROL_VER 942 /* - * change in JSONB format during 9.4 beta + * addition of page header */ #define JSONB_FORMAT_CHANGE_CAT_VER 201409291 @@ -386,6 +396,9 @@ void rewriteVisibilityMap(const char *fromfile, const char *tofile, void check_file_clone(void); void check_hard_link(void); + +int copy_to_new_format(const char *old_subdir, const char *new_subdir, int element_size); + /* fopen_priv() is no longer different from fopen() */ #define fopen_priv(path, mode) fopen(path, mode) @@ -393,6 +406,8 @@ void check_hard_link(void); void get_loadable_libraries(void); void check_loadable_libraries(void); +void cleanup_dirents(struct dirent ** all_dirents, int total_read_files); +struct dirent** get_sorted_hex_files(char * dr, int * size); /* info.c */ diff --git a/src/include/access/nreldefs.h b/src/include/access/nreldefs.h new file mode 100644 index 00000000000..47bca8f48c3 --- /dev/null +++ b/src/include/access/nreldefs.h @@ -0,0 +1,19 @@ +/*------------------------------------------------------------------------- + * + * nreldefs.h + * macros for accessing contents of "nrel" pages + * + * + * Copyright (c) 2021-2022, PostgreSQL Global Development Group + * + * src/include/access/nreldefs.h + * + *-------------------------------------------------------------------------- + */ + +#define SLRU_PAGES_PER_SEGMENT 32 + +#define MULTIXACT_MEMBER_ENTRY_SIZE 20 + +#define MULTIXACT_OFFSET_ENTRY_SIZE 8 + diff --git a/src/include/access/slru.h b/src/include/access/slru.h index 1a9ac3f1924..2cdaedb0f4f 100644 --- a/src/include/access/slru.h +++ b/src/include/access/slru.h @@ -15,6 +15,7 @@ #include "access/xlogdefs.h" + #include "storage/lwlock.h" #include "storage/sync.h" @@ -184,4 +185,5 @@ extern bool SlruScanDirCbDeleteAll(SlruCtl ctl, char *filename, int segpage, void *data); + #endif /* SLRU_H */ diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index fb8561ae0e2..e5ed17633d2 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -47,8 +47,11 @@ typedef enum RBM_ZERO_AND_CLEANUP_LOCK, /* Like RBM_ZERO_AND_LOCK, but locks the page * in "cleanup" mode */ RBM_ZERO_ON_ERROR, /* Read, but return an all-zeros page on error */ - RBM_NORMAL_NO_LOG /* Don't log page as invalid during WAL + RBM_NORMAL_NO_LOG, /* Don't log page as invalid during WAL * replay; otherwise same as RBM_NORMAL */ + + RBM_TRIM /*Read for TRIM functions in CLOG / MultiXact. + Don't validate checksum or zero. */ } ReadBufferMode; /* diff --git a/src/include/storage/bufpage.h b/src/include/storage/bufpage.h index 424ecba028f..a8cd8ce5168 100644 --- a/src/include/storage/bufpage.h +++ b/src/include/storage/bufpage.h @@ -201,6 +201,7 @@ typedef PageHeaderData *PageHeader; * handling pages. */ #define PG_PAGE_LAYOUT_VERSION 4 +#define PG_METAPAGE_LAYOUT_VERSION 1 #define PG_DATA_CHECKSUM_VERSION 1 /* ---------------------------------------------------------------- @@ -302,6 +303,20 @@ PageSetPageSizeAndVersion(Page page, Size size, uint8 version) ((PageHeader) page)->pd_pagesize_version = size | version; } +/* + * PageSetHeaderDataMinimal + * Sets the LSN, page size and version, and checksum + */ +#define PageSetHeaderDataNonRel(page, pageno, lsn, size, version) \ +( \ + PageSetLSN(page, lsn), \ + PageSetPageSizeAndVersion(page, size, version), \ + PageClearHasFreeLinePointers(page), \ + PageSetChecksumInplace(page, pageno) \ +) + + + /* ---------------- * page special data functions * ---------------- @@ -486,6 +501,8 @@ StaticAssertDecl(BLCKSZ == ((BLCKSZ / sizeof(size_t)) * sizeof(size_t)), "BLCKSZ has to be a multiple of sizeof(size_t)"); extern void PageInit(Page page, Size pageSize, Size specialSize); +extern void PageInitNREL(Page page, Size pageSize, Size specialSize); + extern bool PageIsVerifiedExtended(Page page, BlockNumber blkno, int flags); extern OffsetNumber PageAddItemExtended(Page page, Item item, Size size, OffsetNumber offsetNumber, int flags); -- 2.40.1