diff --git a/contrib/pgstattuple/pgstatapprox.c b/contrib/pgstattuple/pgstatapprox.c index 001988b..5d08c73 100644 --- a/contrib/pgstattuple/pgstatapprox.c +++ b/contrib/pgstattuple/pgstatapprox.c @@ -87,7 +87,7 @@ statapprox_heap(Relation rel, output_type *stat) * If the page has only visible tuples, then we can find out the free * space from the FSM and move on. */ - if (visibilitymap_test(rel, blkno, &vmbuffer)) + if (VM_ALL_VISIBLE(rel, blkno, &vmbuffer)) { freespace = GetRecordedFreeSpace(rel, blkno); stat->tuple_len += BLCKSZ - freespace; diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 164d08c..5dc8b04 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -623,18 +623,20 @@ can be used to examine the information stored in free space maps. Each heap relation has a Visibility Map (VM) to keep track of which pages contain only tuples that are known to be -visible to all active transactions. It's stored -alongside the main relation data in a separate relation fork, named after the -filenode number of the relation, plus a _vm suffix. For example, -if the filenode of a relation is 12345, the VM is stored in a file called -12345_vm, in the same directory as the main relation file. +visible to all active transactions, and pages contain only unfrozen tuples. +It's stored alongside the main relation data in a separate relation fork, +named after the filenode number of the relation, plus a _vm suffix. +For example, if the filenode of a relation is 12345, the VM is stored in a file +called 12345_vm, in the same directory as the main relation file. Note that indexes do not have VMs. -The visibility map simply stores one bit per heap page. A set bit means -that all tuples on the page are known to be visible to all transactions. -This means that the page does not contain any tuples that need to be vacuumed. +The visibility map stores two bits per heap page: all-visible, all-frozen. +A set all-visible bit means that all tuples on the page are known to be visible +to all transactions. A set all-frozen bit means that all tuples on the page are +completely marked as frozen. This means that the page does not contain any tuples +that need to be vacuumed, even if a scan of whole table is required. This information can also be used by index-only scans to answer queries using only the index tuple. diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index f443742..5835e54 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -7205,7 +7205,7 @@ log_heap_freeze(Relation reln, Buffer buffer, TransactionId cutoff_xid, */ XLogRecPtr log_heap_visible(RelFileNode rnode, Buffer heap_buffer, Buffer vm_buffer, - TransactionId cutoff_xid) + TransactionId cutoff_xid, uint8 vmflags) { xl_heap_visible xlrec; XLogRecPtr recptr; @@ -7215,6 +7215,7 @@ log_heap_visible(RelFileNode rnode, Buffer heap_buffer, Buffer vm_buffer, Assert(BufferIsValid(vm_buffer)); xlrec.cutoff_xid = cutoff_xid; + xlrec.flags = vmflags; XLogBeginInsert(); XLogRegisterData((char *) &xlrec, SizeOfHeapVisible); @@ -7804,7 +7805,12 @@ heap_xlog_visible(XLogReaderState *record) * the subsequent update won't be replayed to clear the flag. */ page = BufferGetPage(buffer); - PageSetAllVisible(page); + + if (xlrec->flags & VISIBILITYMAP_ALL_VISIBLE) + PageSetAllVisible(page); + if (xlrec->flags & VISIBILITYMAP_ALL_FROZEN) + PageSetAllFrozen(page); + MarkBufferDirty(buffer); } else if (action == BLK_RESTORED) @@ -7856,7 +7862,7 @@ heap_xlog_visible(XLogReaderState *record) */ if (lsn > PageGetLSN(vmpage)) visibilitymap_set(reln, blkno, InvalidBuffer, lsn, vmbuffer, - xlrec->cutoff_xid); + xlrec->cutoff_xid, xlrec->flags); ReleaseBuffer(vmbuffer); FreeFakeRelcacheEntry(reln); diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c index fc28f3f..e269a5d 100644 --- a/src/backend/access/heap/visibilitymap.c +++ b/src/backend/access/heap/visibilitymap.c @@ -15,39 +15,45 @@ * visibilitymap_pin - pin a map page for setting a bit * visibilitymap_pin_ok - check whether correct map page is already pinned * visibilitymap_set - set a bit in a previously pinned page - * visibilitymap_test - test if a bit is set + * visibilitymap_get_status - get status of bits * visibilitymap_count - count number of bits set in visibility map * visibilitymap_truncate - truncate the visibility map * * NOTES * - * The visibility map is a bitmap with one bit per heap page. A set bit means - * that all tuples on the page are known visible to all transactions, and - * therefore the page doesn't need to be vacuumed. The map is conservative in - * the sense that we make sure that whenever a bit is set, we know the - * condition is true, but if a bit is not set, it might or might not be true. + * The visibility map is a bitmap with two bits (all-visible and all-frozen) + * per heap page. A set all-visible bit means that all tuples on the page are + * known visible to all transactions, and therefore the page doesn't need to + * be vacuumed. A set all-frozen bit means that all tuples on the page are + * completely frozen, and therefore the page doesn't need to be vacuumed even + * if whole table scanning vacuum is required (e.g. anti-wraparound vacuum). + * The all-frozen bit must be set only when the page is already all-visible. * - * Clearing a visibility map bit is not separately WAL-logged. The callers + * The map is conservative in the sense that we make sure that whenever a bit + * is set, we know the condition is true, but if a bit is not set, it might or + * might not be true. + * + * Clearing both visibility map bits is not separately WAL-logged. The callers * must make sure that whenever a bit is cleared, the bit is cleared on WAL * replay of the updating operation as well. * * When we *set* a visibility map during VACUUM, we must write WAL. This may * seem counterintuitive, since the bit is basically a hint: if it is clear, - * it may still be the case that every tuple on the page is visible to all - * transactions; we just don't know that for certain. The difficulty is that - * there are two bits which are typically set together: the PD_ALL_VISIBLE bit - * on the page itself, and the visibility map bit. If a crash occurs after the - * visibility map page makes it to disk and before the updated heap page makes - * it to disk, redo must set the bit on the heap page. Otherwise, the next - * insert, update, or delete on the heap page will fail to realize that the - * visibility map bit must be cleared, possibly causing index-only scans to - * return wrong answers. + * it may still be the case that every tuple on the page is all-visible or + * all-frozen we just don't know that for certain. The difficulty is that + * there are two bits which are typically set together: the PD_ALL_VISIBLE + * or PD_ALL_FROZEN bit on the page itself, and the corresponding visibility map + * bit. If a crash occurs after the visibility map page makes it to disk and before + * the updated heap page makes it to disk, redo must set the bit on the heap page. + * Otherwise, the next insert, update, or delete on the heap page will fail to + * realize that the visibility map bit must be cleared, possibly causing index-only + * scans to return wrong answers. * * VACUUM will normally skip pages for which the visibility map bit is set; * such pages can't contain any dead tuples and therefore don't need vacuuming. - * The visibility map is not used for anti-wraparound vacuums, because - * an anti-wraparound vacuum needs to freeze tuples and observe the latest xid - * present in the table, even on pages that don't have any dead tuples. + * The visibility map has the all-frozen bit which indicates all tuples on + * corresponding page have been completely frozen, so the visibility map is also + * used for anti-wraparound vacuum, even if freezing of tuples is required. * * LOCKING * @@ -101,38 +107,50 @@ */ #define MAPSIZE (BLCKSZ - MAXALIGN(SizeOfPageHeaderData)) -/* Number of bits allocated for each heap block. */ -#define BITS_PER_HEAPBLOCK 1 - -/* Number of heap blocks we can represent in one byte. */ -#define HEAPBLOCKS_PER_BYTE 8 - /* Number of heap blocks we can represent in one visibility map page. */ #define HEAPBLOCKS_PER_PAGE (MAPSIZE * HEAPBLOCKS_PER_BYTE) /* Mapping from heap block number to the right bit in the visibility map */ #define HEAPBLK_TO_MAPBLOCK(x) ((x) / HEAPBLOCKS_PER_PAGE) #define HEAPBLK_TO_MAPBYTE(x) (((x) % HEAPBLOCKS_PER_PAGE) / HEAPBLOCKS_PER_BYTE) -#define HEAPBLK_TO_MAPBIT(x) ((x) % HEAPBLOCKS_PER_BYTE) - -/* table for fast counting of set bits */ -static const uint8 number_of_ones[256] = { - 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, - 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, - 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, - 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, - 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, - 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, - 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, - 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, - 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, - 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8 +#define HEAPBLK_TO_MAPBIT(x) (((x) % HEAPBLOCKS_PER_BYTE) * BITS_PER_HEAPBLOCK) + +/* tables for fast counting of set bits for visible and frozen */ +static const uint8 number_of_ones_for_visible[256] = { + 0, 1, 0, 1, 1, 2, 1, 2, 0, 1, 0, 1, 1, 2, 1, 2, + 1, 2, 1, 2, 2, 3, 2, 3, 1, 2, 1, 2, 2, 3, 2, 3, + 0, 1, 0, 1, 1, 2, 1, 2, 0, 1, 0, 1, 1, 2, 1, 2, + 1, 2, 1, 2, 2, 3, 2, 3, 1, 2, 1, 2, 2, 3, 2, 3, + 1, 2, 1, 2, 2, 3, 2, 3, 1, 2, 1, 2, 2, 3, 2, 3, + 2, 3, 2, 3, 3, 4, 3, 4, 2, 3, 2, 3, 3, 4, 3, 4, + 1, 2, 1, 2, 2, 3, 2, 3, 1, 2, 1, 2, 2, 3, 2, 3, + 2, 3, 2, 3, 3, 4, 3, 4, 2, 3, 2, 3, 3, 4, 3, 4, + 0, 1, 0, 1, 1, 2, 1, 2, 0, 1, 0, 1, 1, 2, 1, 2, + 1, 2, 1, 2, 2, 3, 2, 3, 1, 2, 1, 2, 2, 3, 2, 3, + 0, 1, 0, 1, 1, 2, 1, 2, 0, 1, 0, 1, 1, 2, 1, 2, + 1, 2, 1, 2, 2, 3, 2, 3, 1, 2, 1, 2, 2, 3, 2, 3, + 1, 2, 1, 2, 2, 3, 2, 3, 1, 2, 1, 2, 2, 3, 2, 3, + 2, 3, 2, 3, 3, 4, 3, 4, 2, 3, 2, 3, 3, 4, 3, 4, + 1, 2, 1, 2, 2, 3, 2, 3, 1, 2, 1, 2, 2, 3, 2, 3, + 2, 3, 2, 3, 3, 4, 3, 4, 2, 3, 2, 3, 3, 4, 3, 4 +}; +static const uint8 number_of_ones_for_frozen[256] = { + 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 2, 2, 1, 1, 2, 2, + 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 2, 2, 1, 1, 2, 2, + 1, 1, 2, 2, 1, 1, 2, 2, 2, 2, 3, 3, 2, 2, 3, 3, + 1, 1, 2, 2, 1, 1, 2, 2, 2, 2, 3, 3, 2, 2, 3, 3, + 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 2, 2, 1, 1, 2, 2, + 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 2, 2, 1, 1, 2, 2, + 1, 1, 2, 2, 1, 1, 2, 2, 2, 2, 3, 3, 2, 2, 3, 3, + 1, 1, 2, 2, 1, 1, 2, 2, 2, 2, 3, 3, 2, 2, 3, 3, + 1, 1, 2, 2, 1, 1, 2, 2, 2, 2, 3, 3, 2, 2, 3, 3, + 1, 1, 2, 2, 1, 1, 2, 2, 2, 2, 3, 3, 2, 2, 3, 3, + 2, 2, 3, 3, 2, 2, 3, 3, 3, 3, 4, 4, 3, 3, 4, 4, + 2, 2, 3, 3, 2, 2, 3, 3, 3, 3, 4, 4, 3, 3, 4, 4, + 1, 1, 2, 2, 1, 1, 2, 2, 2, 2, 3, 3, 2, 2, 3, 3, + 1, 1, 2, 2, 1, 1, 2, 2, 2, 2, 3, 3, 2, 2, 3, 3, + 2, 2, 3, 3, 2, 2, 3, 3, 3, 3, 4, 4, 3, 3, 4, 4, + 2, 2, 3, 3, 2, 2, 3, 3, 3, 3, 4, 4, 3, 3, 4, 4 }; /* prototypes for internal routines */ @@ -141,7 +159,7 @@ static void vm_extend(Relation rel, BlockNumber nvmblocks); /* - * visibilitymap_clear - clear a bit in visibility map + * visibilitymap_clear - clear all bits in visibility map * * You must pass a buffer containing the correct map page to this function. * Call visibilitymap_pin first to pin the right one. This function doesn't do @@ -153,7 +171,7 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer buf) BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk); int mapByte = HEAPBLK_TO_MAPBYTE(heapBlk); int mapBit = HEAPBLK_TO_MAPBIT(heapBlk); - uint8 mask = 1 << mapBit; + uint8 mask = VISIBILITYMAP_VALID_BITS << mapBit; char *map; #ifdef TRACE_VISIBILITYMAP @@ -186,7 +204,7 @@ visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer buf) * visibilitymap_set to actually set the bit. * * On entry, *buf should be InvalidBuffer or a valid buffer returned by - * an earlier call to visibilitymap_pin or visibilitymap_test on the same + * an earlier call to visibilitymap_pin or visibilitymap_get_status on the same * relation. On return, *buf is a valid buffer with the map page containing * the bit for heapBlk. * @@ -212,7 +230,7 @@ visibilitymap_pin(Relation rel, BlockNumber heapBlk, Buffer *buf) * visibilitymap_pin_ok - do we already have the correct page pinned? * * On entry, buf should be InvalidBuffer or a valid buffer returned by - * an earlier call to visibilitymap_pin or visibilitymap_test on the same + * an earlier call to visibilitymap_pin or visibilitymap_get_status on the same * relation. The return value indicates whether the buffer covers the * given heapBlk. */ @@ -225,7 +243,7 @@ visibilitymap_pin_ok(BlockNumber heapBlk, Buffer buf) } /* - * visibilitymap_set - set a bit on a previously pinned page + * visibilitymap_set - set bit(s) on a previously pinned page * * recptr is the LSN of the XLOG record we're replaying, if we're in recovery, * or InvalidXLogRecPtr in normal running. The page LSN is advanced to the @@ -234,10 +252,11 @@ visibilitymap_pin_ok(BlockNumber heapBlk, Buffer buf) * marked all-visible; it is needed for Hot Standby, and can be * InvalidTransactionId if the page contains no tuples. * - * Caller is expected to set the heap page's PD_ALL_VISIBLE bit before calling - * this function. Except in recovery, caller should also pass the heap - * buffer. When checksums are enabled and we're not in recovery, we must add - * the heap buffer to the WAL chain to protect it from being torn. + * Caller is expected to set the heap page's PD_ALL_VISIBLE or PD_ALL_FROZEN + * bit before calling this function. Except in recovery, caller should also + * pass the heap buffer and flags which indicates what flag we want to set. + * When checksums are enabled and we're not in recovery, we must add the heap + * buffer to the WAL chain to protect it from being torn. * * You must pass a buffer containing the correct map page to this function. * Call visibilitymap_pin first to pin the right one. This function doesn't do @@ -245,13 +264,14 @@ visibilitymap_pin_ok(BlockNumber heapBlk, Buffer buf) */ void visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf, - XLogRecPtr recptr, Buffer vmBuf, TransactionId cutoff_xid) + XLogRecPtr recptr, Buffer vmBuf, TransactionId cutoff_xid, + uint8 flags) { BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk); uint32 mapByte = HEAPBLK_TO_MAPBYTE(heapBlk); uint8 mapBit = HEAPBLK_TO_MAPBIT(heapBlk); Page page; - char *map; + uint8 *map; #ifdef TRACE_VISIBILITYMAP elog(DEBUG1, "vm_set %s %d", RelationGetRelationName(rel), heapBlk); @@ -259,6 +279,7 @@ visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf, Assert(InRecovery || XLogRecPtrIsInvalid(recptr)); Assert(InRecovery || BufferIsValid(heapBuf)); + Assert(flags & VISIBILITYMAP_VALID_BITS); /* Check that we have the right heap page pinned, if present */ if (BufferIsValid(heapBuf) && BufferGetBlockNumber(heapBuf) != heapBlk) @@ -269,14 +290,14 @@ visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf, elog(ERROR, "wrong VM buffer passed to visibilitymap_set"); page = BufferGetPage(vmBuf); - map = PageGetContents(page); + map = (uint8 *)PageGetContents(page); LockBuffer(vmBuf, BUFFER_LOCK_EXCLUSIVE); - if (!(map[mapByte] & (1 << mapBit))) + if (flags != (map[mapByte] >> mapBit & VISIBILITYMAP_VALID_BITS)) { START_CRIT_SECTION(); - map[mapByte] |= (1 << mapBit); + map[mapByte] |= (flags << mapBit); MarkBufferDirty(vmBuf); if (RelationNeedsWAL(rel)) @@ -285,7 +306,7 @@ visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf, { Assert(!InRecovery); recptr = log_heap_visible(rel->rd_node, heapBuf, vmBuf, - cutoff_xid); + cutoff_xid, flags); /* * If data checksums are enabled (or wal_log_hints=on), we @@ -295,11 +316,19 @@ visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf, { Page heapPage = BufferGetPage(heapBuf); - /* caller is expected to set PD_ALL_VISIBLE first */ - Assert(PageIsAllVisible(heapPage)); + /* + * Caller is expected to set PD_ALL_VISIBLE or + * PD_ALL_FROZEN first. + */ + if (flags & VISIBILITYMAP_ALL_VISIBLE) + Assert(PageIsAllVisible(heapPage)); + if (flags & VISIBILITYMAP_ALL_FROZEN) + Assert(PageIsAllFrozen(heapPage)); + PageSetLSN(heapPage, recptr); } } + PageSetLSN(page, recptr); } @@ -310,15 +339,17 @@ visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf, } /* - * visibilitymap_test - test if a bit is set + * visibilitymap_get_status - get status of bits * - * Are all tuples on heapBlk visible to all, according to the visibility map? + * Are all tuples on heapBlk visible to all or are marked frozen, according + * to the visibility map? * * On entry, *buf should be InvalidBuffer or a valid buffer returned by an - * earlier call to visibilitymap_pin or visibilitymap_test on the same + * earlier call to visibilitymap_pin or visibilitymap_get_status on the same * relation. On return, *buf is a valid buffer with the map page containing * the bit for heapBlk, or InvalidBuffer. The caller is responsible for - * releasing *buf after it's done testing and setting bits. + * releasing *buf after it's done testing and setting bits, and must pass flags + * for which it needs to check the value in visibility map. * * NOTE: This function is typically called without a lock on the heap page, * so somebody else could change the bit just after we look at it. In fact, @@ -327,17 +358,16 @@ visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf, * we might see the old value. It is the caller's responsibility to deal with * all concurrency issues! */ -bool -visibilitymap_test(Relation rel, BlockNumber heapBlk, Buffer *buf) +uint8 +visibilitymap_get_status(Relation rel, BlockNumber heapBlk, Buffer *buf) { BlockNumber mapBlock = HEAPBLK_TO_MAPBLOCK(heapBlk); uint32 mapByte = HEAPBLK_TO_MAPBYTE(heapBlk); uint8 mapBit = HEAPBLK_TO_MAPBIT(heapBlk); - bool result; char *map; #ifdef TRACE_VISIBILITYMAP - elog(DEBUG1, "vm_test %s %d", RelationGetRelationName(rel), heapBlk); + elog(DEBUG1, "vm_get_status %s %d", RelationGetRelationName(rel), heapBlk); #endif /* Reuse the old pinned buffer if possible */ @@ -360,13 +390,11 @@ visibilitymap_test(Relation rel, BlockNumber heapBlk, Buffer *buf) map = PageGetContents(BufferGetPage(*buf)); /* - * A single-bit read is atomic. There could be memory-ordering effects + * The double bits read is atomic. There could be memory-ordering effects * here, but for performance reasons we make it the caller's job to worry * about that. */ - result = (map[mapByte] & (1 << mapBit)) ? true : false; - - return result; + return ((map[mapByte] >> mapBit) & VISIBILITYMAP_VALID_BITS); } /* @@ -374,14 +402,20 @@ visibilitymap_test(Relation rel, BlockNumber heapBlk, Buffer *buf) * * Note: we ignore the possibility of race conditions when the table is being * extended concurrently with the call. New pages added to the table aren't - * going to be marked all-visible, so they won't affect the result. + * going to be marked all-visible or all-frozen, so they won't affect the result. */ -BlockNumber -visibilitymap_count(Relation rel) +void +visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_frozen) { - BlockNumber result = 0; BlockNumber mapBlock; + /* all_visible must be specified */ + Assert(all_visible); + + *all_visible = 0; + if (all_frozen) + *all_frozen = 0; + for (mapBlock = 0;; mapBlock++) { Buffer mapBuffer; @@ -406,13 +440,13 @@ visibilitymap_count(Relation rel) for (i = 0; i < MAPSIZE; i++) { - result += number_of_ones[map[i]]; + *all_visible += number_of_ones_for_visible[map[i]]; + if (all_frozen) + *all_frozen += number_of_ones_for_frozen[map[i]]; } ReleaseBuffer(mapBuffer); } - - return result; } /* diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index a309c44..3737b10 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -1920,7 +1920,7 @@ index_update_stats(Relation rel, BlockNumber relallvisible; if (rd_rel->relkind != RELKIND_INDEX) - relallvisible = visibilitymap_count(rel); + visibilitymap_count(rel, &relallvisible, NULL); else /* don't bother for indexes */ relallvisible = 0; diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 923fe58..86437c6 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -452,6 +452,7 @@ CREATE VIEW pg_stat_all_tables AS pg_stat_get_live_tuples(C.oid) AS n_live_tup, pg_stat_get_dead_tuples(C.oid) AS n_dead_tup, pg_stat_get_mod_since_analyze(C.oid) AS n_mod_since_analyze, + pg_stat_get_frozen_pages(C.oid) AS n_frozen_pages, pg_stat_get_last_vacuum_time(C.oid) as last_vacuum, pg_stat_get_last_autovacuum_time(C.oid) as last_autovacuum, pg_stat_get_last_analyze_time(C.oid) as last_analyze, diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index 070df29..d7f3035 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -314,6 +314,8 @@ do_analyze_rel(Relation onerel, int options, VacuumParams *params, Oid save_userid; int save_sec_context; int save_nestlevel; + BlockNumber relallvisible, + relallfrozen; if (inh) ereport(elevel, @@ -564,51 +566,56 @@ do_analyze_rel(Relation onerel, int options, VacuumParams *params, } } - /* - * Update pages/tuples stats in pg_class ... but not if we're doing - * inherited stats. - */ if (!inh) + { + /* Calculate the number of all-visible and all-frozen bit */ + visibilitymap_count(onerel, &relallvisible, &relallfrozen); + + /* + * Update pages/tuples stats in pg_class ... but not if we're doing + * inherited stats. + */ vac_update_relstats(onerel, relpages, totalrows, - visibilitymap_count(onerel), + relallvisible, hasindex, InvalidTransactionId, InvalidMultiXactId, in_outer_xact); - /* - * Same for indexes. Vacuum always scans all indexes, so if we're part of - * VACUUM ANALYZE, don't overwrite the accurate count already inserted by - * VACUUM. - */ - if (!inh && !(options & VACOPT_VACUUM)) - { - for (ind = 0; ind < nindexes; ind++) + /* + * Same for indexes. Vacuum always scans all indexes, so if we're part of + * VACUUM ANALYZE, don't overwrite the accurate count already inserted by + * VACUUM. + */ + if (!(options & VACOPT_VACUUM)) { - AnlIndexData *thisdata = &indexdata[ind]; - double totalindexrows; - - totalindexrows = ceil(thisdata->tupleFract * totalrows); - vac_update_relstats(Irel[ind], - RelationGetNumberOfBlocks(Irel[ind]), - totalindexrows, - 0, - false, - InvalidTransactionId, - InvalidMultiXactId, - in_outer_xact); + for (ind = 0; ind < nindexes; ind++) + { + AnlIndexData *thisdata = &indexdata[ind]; + double totalindexrows; + + totalindexrows = ceil(thisdata->tupleFract * totalrows); + vac_update_relstats(Irel[ind], + RelationGetNumberOfBlocks(Irel[ind]), + totalindexrows, + 0, + false, + InvalidTransactionId, + InvalidMultiXactId, + in_outer_xact); + } } - } - /* - * Report ANALYZE to the stats collector, too. However, if doing - * inherited stats we shouldn't report, because the stats collector only - * tracks per-table stats. - */ - if (!inh) - pgstat_report_analyze(onerel, totalrows, totaldeadrows); + /* + * Report ANALYZE to the stats collector, too. However, if doing + * inherited stats we shouldn't report, because the stats collector only + * tracks per-table stats. + */ + pgstat_report_analyze(onerel, totalrows, totaldeadrows, relallfrozen); + + } /* If this isn't part of VACUUM ANALYZE, let index AMs do cleanup */ if (!(options & VACOPT_VACUUM)) diff --git a/src/backend/executor/nodeIndexonlyscan.c b/src/backend/executor/nodeIndexonlyscan.c index 90afbdc..4f6f91c 100644 --- a/src/backend/executor/nodeIndexonlyscan.c +++ b/src/backend/executor/nodeIndexonlyscan.c @@ -85,9 +85,9 @@ IndexOnlyNext(IndexOnlyScanState *node) * which all tuples are known visible to everybody. In any case, * we'll use the index tuple not the heap tuple as the data source. * - * Note on Memory Ordering Effects: visibilitymap_test does not lock - * the visibility map buffer, and therefore the result we read here - * could be slightly stale. However, it can't be stale enough to + * Note on Memory Ordering Effects: visibilitymap_get_status does not + * lock the visibility map buffer, and therefore the result we read + * here could be slightly stale. However, it can't be stale enough to * matter. * * We need to detect clearing a VM bit due to an insert right away, @@ -114,9 +114,9 @@ IndexOnlyNext(IndexOnlyScanState *node) * It's worth going through this complexity to avoid needing to lock * the VM buffer, which could cause significant contention. */ - if (!visibilitymap_test(scandesc->heapRelation, - ItemPointerGetBlockNumber(tid), - &node->ioss_VMBuffer)) + if (!VM_ALL_VISIBLE(scandesc->heapRelation, + ItemPointerGetBlockNumber(tid), + &node->ioss_VMBuffer)) { /* * Rats, we have to visit the heap to check visibility. diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index da768c6..08b61cb 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -1329,7 +1329,8 @@ pgstat_report_autovac(Oid dboid) */ void pgstat_report_vacuum(Oid tableoid, bool shared, - PgStat_Counter livetuples, PgStat_Counter deadtuples) + PgStat_Counter livetuples, PgStat_Counter deadtuples, + int32 frozenpages) { PgStat_MsgVacuum msg; @@ -1343,6 +1344,7 @@ pgstat_report_vacuum(Oid tableoid, bool shared, msg.m_vacuumtime = GetCurrentTimestamp(); msg.m_live_tuples = livetuples; msg.m_dead_tuples = deadtuples; + msg.m_frozen_pages = frozenpages; pgstat_send(&msg, sizeof(msg)); } @@ -1354,7 +1356,8 @@ pgstat_report_vacuum(Oid tableoid, bool shared, */ void pgstat_report_analyze(Relation rel, - PgStat_Counter livetuples, PgStat_Counter deadtuples) + PgStat_Counter livetuples, PgStat_Counter deadtuples, + int32 frozenpages) { PgStat_MsgAnalyze msg; @@ -1394,6 +1397,7 @@ pgstat_report_analyze(Relation rel, msg.m_analyzetime = GetCurrentTimestamp(); msg.m_live_tuples = livetuples; msg.m_dead_tuples = deadtuples; + msg.m_frozen_pages = frozenpages; pgstat_send(&msg, sizeof(msg)); } @@ -3702,6 +3706,7 @@ pgstat_get_tab_entry(PgStat_StatDBEntry *dbentry, Oid tableoid, bool create) result->n_live_tuples = 0; result->n_dead_tuples = 0; result->changes_since_analyze = 0; + result->n_frozen_pages = 0; result->blocks_fetched = 0; result->blocks_hit = 0; result->vacuum_timestamp = 0; @@ -5069,6 +5074,7 @@ pgstat_recv_vacuum(PgStat_MsgVacuum *msg, int len) tabentry->n_live_tuples = msg->m_live_tuples; tabentry->n_dead_tuples = msg->m_dead_tuples; + tabentry->n_frozen_pages = msg->m_frozen_pages; if (msg->m_autovacuum) { @@ -5103,6 +5109,7 @@ pgstat_recv_analyze(PgStat_MsgAnalyze *msg, int len) tabentry->n_live_tuples = msg->m_live_tuples; tabentry->n_dead_tuples = msg->m_dead_tuples; + tabentry->n_frozen_pages = msg->m_frozen_pages; /* * We reset changes_since_analyze to zero, forgetting any changes that diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index 1b22fcc..7c57b3e 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -46,6 +46,7 @@ extern Datum pg_stat_get_vacuum_count(PG_FUNCTION_ARGS); extern Datum pg_stat_get_autovacuum_count(PG_FUNCTION_ARGS); extern Datum pg_stat_get_analyze_count(PG_FUNCTION_ARGS); extern Datum pg_stat_get_autoanalyze_count(PG_FUNCTION_ARGS); +extern Datum pg_stat_get_frozen_pages(PG_FUNCTION_ARGS); extern Datum pg_stat_get_function_calls(PG_FUNCTION_ARGS); extern Datum pg_stat_get_function_total_time(PG_FUNCTION_ARGS); @@ -450,6 +451,21 @@ pg_stat_get_autoanalyze_count(PG_FUNCTION_ARGS) } Datum +pg_stat_get_frozen_pages(PG_FUNCTION_ARGS) +{ + Oid relid = PG_GETARG_OID(0); + int32 result; + PgStat_StatTabEntry *tabentry; + + if ((tabentry = pgstat_fetch_stat_tabentry(relid)) == NULL) + result = 0; + else + result = (int32) (tabentry->n_frozen_pages); + + PG_RETURN_INT32(result); +} + +Datum pg_stat_get_function_calls(PG_FUNCTION_ARGS) { Oid funcid = PG_GETARG_OID(0); diff --git a/src/include/access/heapam_xlog.h b/src/include/access/heapam_xlog.h index f77489b..583b55a 100644 --- a/src/include/access/heapam_xlog.h +++ b/src/include/access/heapam_xlog.h @@ -320,9 +320,10 @@ typedef struct xl_heap_freeze_page typedef struct xl_heap_visible { TransactionId cutoff_xid; + uint8 flags; } xl_heap_visible; -#define SizeOfHeapVisible (offsetof(xl_heap_visible, cutoff_xid) + sizeof(TransactionId)) +#define SizeOfHeapVisible (offsetof(xl_heap_visible, flags) + sizeof(uint8)) typedef struct xl_heap_new_cid { @@ -389,6 +390,6 @@ extern bool heap_prepare_freeze_tuple(HeapTupleHeader tuple, extern void heap_execute_freeze_tuple(HeapTupleHeader tuple, xl_heap_freeze_tuple *xlrec_tp); extern XLogRecPtr log_heap_visible(RelFileNode rnode, Buffer heap_buffer, - Buffer vm_buffer, TransactionId cutoff_xid); + Buffer vm_buffer, TransactionId cutoff_xid, uint8 flags); #endif /* HEAPAM_XLOG_H */ diff --git a/src/include/access/visibilitymap.h b/src/include/access/visibilitymap.h index d447daf..a75de5c 100644 --- a/src/include/access/visibilitymap.h +++ b/src/include/access/visibilitymap.h @@ -19,15 +19,36 @@ #include "storage/buf.h" #include "utils/relcache.h" +/* + * Number of bits allocated for each heap block. + * One for all-visible, other for all-frozen. +*/ +#define BITS_PER_HEAPBLOCK 2 + +/* Number of heap blocks we can represent in one byte. */ +#define HEAPBLOCKS_PER_BYTE 4 + +/* Flags for bit map */ +#define VISIBILITYMAP_ALL_VISIBLE 0x01 +#define VISIBILITYMAP_ALL_FROZEN 0x02 +#define VISIBILITYMAP_VALID_BITS 0x03 /* OR of all valid visiblitymap flags bits */ + +/* Macros for visibilitymap test */ +#define VM_ALL_VISIBLE(r, b, v) \ + ((visibilitymap_get_status((r), (b), (v)) & VISIBILITYMAP_ALL_VISIBLE) != 0) +#define VM_ALL_FROZEN(r, b, v) \ + ((visibilitymap_get_status((r), (b), (v)) & VISIBILITYMAP_ALL_FROZEN) != 0) + extern void visibilitymap_clear(Relation rel, BlockNumber heapBlk, Buffer vmbuf); extern void visibilitymap_pin(Relation rel, BlockNumber heapBlk, Buffer *vmbuf); extern bool visibilitymap_pin_ok(BlockNumber heapBlk, Buffer vmbuf); extern void visibilitymap_set(Relation rel, BlockNumber heapBlk, Buffer heapBuf, - XLogRecPtr recptr, Buffer vmBuf, TransactionId cutoff_xid); -extern bool visibilitymap_test(Relation rel, BlockNumber heapBlk, Buffer *vmbuf); -extern BlockNumber visibilitymap_count(Relation rel); + XLogRecPtr recptr, Buffer vmBuf, TransactionId cutoff_xid, + uint8 flags); +extern uint8 visibilitymap_get_status(Relation rel, BlockNumber heapBlk, Buffer *vmbuf); +extern void visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_frozen); extern void visibilitymap_truncate(Relation rel, BlockNumber nheapblocks); #endif /* VISIBILITYMAP_H */ diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 378c40f..a0b210b 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -53,6 +53,6 @@ */ /* yyyymmddN */ -#define CATALOG_VERSION_NO 201602071 +#define CATALOG_VERSION_NO 201602131 #endif diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 1c0ef9a..a0420b5 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -2706,6 +2706,8 @@ DATA(insert OID = 3056 ( pg_stat_get_analyze_count PGNSP PGUID 12 1 0 0 0 f f f DESCR("statistics: number of manual analyzes for a table"); DATA(insert OID = 3057 ( pg_stat_get_autoanalyze_count PGNSP PGUID 12 1 0 0 0 f f f f t f s r 1 0 20 "26" _null_ _null_ _null_ _null_ _null_ pg_stat_get_autoanalyze_count _null_ _null_ _null_ )); DESCR("statistics: number of auto analyzes for a table"); +DATA(insert OID = 6015 ( pg_stat_get_frozen_pages PGNSP PGUID 12 1 0 0 0 f f f f t f s r 1 0 23 "26" _null_ _null_ _null_ _null_ _null_ pg_stat_get_frozen_pages _null_ _null_ _null_ )); +DESCR("statistics: number of frozen pages of table"); DATA(insert OID = 1936 ( pg_stat_get_backend_idset PGNSP PGUID 12 1 100 0 0 f f f f t t s r 0 0 23 "" _null_ _null_ _null_ _null_ _null_ pg_stat_get_backend_idset _null_ _null_ _null_ )); DESCR("statistics: currently active backend IDs"); DATA(insert OID = 2022 ( pg_stat_get_activity PGNSP PGUID 12 1 100 0 0 f f f f f t s r 1 0 2249 "23" "{23,26,23,26,25,25,25,16,1184,1184,1184,1184,869,25,23,28,28,16,25,25,23,16,25}" "{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}" "{pid,datid,pid,usesysid,application_name,state,query,waiting,xact_start,query_start,backend_start,state_change,client_addr,client_hostname,client_port,backend_xid,backend_xmin,ssl,sslversion,sslcipher,sslbits,sslcompression,sslclientdn}" _null_ _null_ pg_stat_get_activity _null_ _null_ _null_ )); diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 65e968e..ad40b70 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -17,6 +17,7 @@ #include "portability/instr_time.h" #include "postmaster/pgarch.h" #include "storage/barrier.h" +#include "storage/block.h" #include "utils/hsearch.h" #include "utils/relcache.h" @@ -355,6 +356,7 @@ typedef struct PgStat_MsgVacuum TimestampTz m_vacuumtime; PgStat_Counter m_live_tuples; PgStat_Counter m_dead_tuples; + BlockNumber m_frozen_pages; } PgStat_MsgVacuum; @@ -372,6 +374,7 @@ typedef struct PgStat_MsgAnalyze TimestampTz m_analyzetime; PgStat_Counter m_live_tuples; PgStat_Counter m_dead_tuples; + int32 m_frozen_pages; } PgStat_MsgAnalyze; @@ -551,7 +554,7 @@ typedef union PgStat_Msg * ------------------------------------------------------------ */ -#define PGSTAT_FILE_FORMAT_ID 0x01A5BC9D +#define PGSTAT_FILE_FORMAT_ID 0x01A5BC9E /* ---------- * PgStat_StatDBEntry The collector's data per database @@ -615,6 +618,8 @@ typedef struct PgStat_StatTabEntry PgStat_Counter n_dead_tuples; PgStat_Counter changes_since_analyze; + int32 n_frozen_pages; + PgStat_Counter blocks_fetched; PgStat_Counter blocks_hit; @@ -917,9 +922,11 @@ extern void pgstat_reset_single_counter(Oid objectid, PgStat_Single_Reset_Type t extern void pgstat_report_autovac(Oid dboid); extern void pgstat_report_vacuum(Oid tableoid, bool shared, - PgStat_Counter livetuples, PgStat_Counter deadtuples); + PgStat_Counter livetuples, PgStat_Counter deadtuples, + int32 frozenpages); extern void pgstat_report_analyze(Relation rel, - PgStat_Counter livetuples, PgStat_Counter deadtuples); + PgStat_Counter livetuples, PgStat_Counter deadtuples, + int32 frozenpages); extern void pgstat_report_recovery_conflict(int reason); extern void pgstat_report_deadlock(void); diff --git a/src/include/storage/bufpage.h b/src/include/storage/bufpage.h index 2ce3be7..0b023b3 100644 --- a/src/include/storage/bufpage.h +++ b/src/include/storage/bufpage.h @@ -178,8 +178,10 @@ typedef PageHeaderData *PageHeader; * tuple? */ #define PD_ALL_VISIBLE 0x0004 /* all tuples on page are visible to * everyone */ +#define PD_ALL_FROZEN 0x0008 /* all tuples on page are completely + frozen */ -#define PD_VALID_FLAG_BITS 0x0007 /* OR of all valid pd_flags bits */ +#define PD_VALID_FLAG_BITS 0x000F /* OR of all valid pd_flags bits */ /* * Page layout version number 0 is for pre-7.3 Postgres releases. @@ -367,7 +369,12 @@ typedef PageHeaderData *PageHeader; #define PageSetAllVisible(page) \ (((PageHeader) (page))->pd_flags |= PD_ALL_VISIBLE) #define PageClearAllVisible(page) \ - (((PageHeader) (page))->pd_flags &= ~PD_ALL_VISIBLE) + (((PageHeader) (page))->pd_flags &= ~(PD_ALL_VISIBLE | PD_ALL_FROZEN)) + +#define PageIsAllFrozen(page) \ + (((PageHeader) (page))->pd_flags & PD_ALL_FROZEN) +#define PageSetAllFrozen(page) \ + (((PageHeader) (page))->pd_flags |= PD_ALL_FROZEN) #define PageIsPrunable(page, oldestxmin) \ ( \ diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 2bdba2d..0f13ab0 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1683,6 +1683,7 @@ pg_stat_all_tables| SELECT c.oid AS relid, pg_stat_get_live_tuples(c.oid) AS n_live_tup, pg_stat_get_dead_tuples(c.oid) AS n_dead_tup, pg_stat_get_mod_since_analyze(c.oid) AS n_mod_since_analyze, + pg_stat_get_frozen_pages(c.oid) AS n_frozen_pages, pg_stat_get_last_vacuum_time(c.oid) AS last_vacuum, pg_stat_get_last_autovacuum_time(c.oid) AS last_autovacuum, pg_stat_get_last_analyze_time(c.oid) AS last_analyze, @@ -1795,6 +1796,7 @@ pg_stat_sys_tables| SELECT pg_stat_all_tables.relid, pg_stat_all_tables.n_live_tup, pg_stat_all_tables.n_dead_tup, pg_stat_all_tables.n_mod_since_analyze, + pg_stat_all_tables.n_frozen_pages, pg_stat_all_tables.last_vacuum, pg_stat_all_tables.last_autovacuum, pg_stat_all_tables.last_analyze, @@ -1838,6 +1840,7 @@ pg_stat_user_tables| SELECT pg_stat_all_tables.relid, pg_stat_all_tables.n_live_tup, pg_stat_all_tables.n_dead_tup, pg_stat_all_tables.n_mod_since_analyze, + pg_stat_all_tables.n_frozen_pages, pg_stat_all_tables.last_vacuum, pg_stat_all_tables.last_autovacuum, pg_stat_all_tables.last_analyze, diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index f5be70f..95ababf 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -127,6 +127,8 @@ SELECT count(*) FROM tenk2 WHERE unique1 = 1; 1 (1 row) +-- do VACUUM FREEZE +VACUUM FREEZE tenk2; -- force the rate-limiting logic in pgstat_report_tabstat() to time out -- and send a message SELECT pg_sleep(1.0); @@ -175,6 +177,14 @@ SELECT st.heap_blks_read + st.heap_blks_hit >= pr.heap_blks + cl.relpages, t | t (1 row) +SELECT n_frozen_pages = (pg_relation_size('tenk2') / current_setting('block_size')::int) + FROM pg_stat_user_tables + WHERE relname ='tenk2'; + ?column? +---------- + t +(1 row) + SELECT pr.snap_ts < pg_stat_get_snapshot_timestamp() as snapshot_newer FROM prevstats AS pr; snapshot_newer diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index cd2d592..dea5553 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -120,6 +120,8 @@ ROLLBACK; SELECT count(*) FROM tenk2; -- do an indexscan SELECT count(*) FROM tenk2 WHERE unique1 = 1; +-- do VACUUM FREEZE +VACUUM FREEZE tenk2; -- force the rate-limiting logic in pgstat_report_tabstat() to time out -- and send a message @@ -145,6 +147,10 @@ SELECT st.heap_blks_read + st.heap_blks_hit >= pr.heap_blks + cl.relpages, FROM pg_statio_user_tables AS st, pg_class AS cl, prevstats AS pr WHERE st.relname='tenk2' AND cl.relname='tenk2'; +SELECT n_frozen_pages = (pg_relation_size('tenk2') / current_setting('block_size')::int) + FROM pg_stat_user_tables + WHERE relname ='tenk2'; + SELECT pr.snap_ts < pg_stat_get_snapshot_timestamp() as snapshot_newer FROM prevstats AS pr;