From 63a1b25eb6d3c93c453b4e6dc5c4ef5c67e3de8a Mon Sep 17 00:00:00 2001 From: David Christensen Date: Thu, 1 Jun 2023 11:34:14 -0400 Subject: [PATCH v2 1/5] Introduce initdb-selectable block sizes Whereas we had traditionally used BLCKSZ to indicate the compile-time selected block size, this commit adjusted things so the cluster block size can be selected at initdb time. In order to code for this, we introduce a few new defines: - cluster_block_size is the blocksize for this cluster itself. This is not valid until BlockSizeInit() has been called in the given backend, which we do as early as possible by parsing the ControlFile and using the blcksz field. - MIN_BLOCK_SIZE and MAX_BLOCK_SIZE are the limits for the selectable block size. It is required that cluster_block_size is a power of 2 between these two constants. - DEFAULT_BLOCK_SIZE is the moral equivalent of BLCKSZ; it is the built-in default value. This is used in a few places that just needed a buffer of an arbitrary size, but the dynamic value cluster_block_size should almost always be used instead. - cluster_relseg_size is used instead of RELSEG_SIZE, since internally we are storing the segment size in terms of number of blocks. RELSEG_SIZE is still kept, but is used in terms of the number of blocks of DEFAULT_BLOCK_SIZE; cluster_relseg_size scales appropriately (and is the only thing used internally) to keep the same target total segment size regardless of block size. This patch uses inline functions to switch between compile-time constants for all blocksize-derived fields; this makes what could be a heavy computation or memory load/store in hot code paths a relatively fast (~ 4 instructions) computation for any access to such a dynamic value. Where possible, the traditional macro names are kept so code that doesn't care about it should not need to change, however the definition of these has changed (see the CalcXXX() routines in blocksize.h for details). A new function, BlockSizeInit() populates the appropriate values based on the target block size. This should be called as early as possible in any code that utilizes block sizes. This patch adds this in the appropriate place on the handful of src/bin/ programs that used cluster_block_size, so this caveat mainly impacts new code. Headers which define now dynamic values should declare the symbol name and the calculation macro in order to properly register it; BlockSizeDecl() ends up creating the necessary static inline function, but only in files which include the given headers. Code which had previously used BLCKZ should likely be able to get away with changing these instances to cluster_block_size, unless you're using a structure allocated on the stack. In these cases, the compiler will complain about dynamic structure. The solution is to utilize an expression with MAX_BLOCK_SIZE instead of cluster_block_size, ensuring enough stack space is allocated for the maximum size. This also does require using cluster_block_size or an expression based on it when actually using this structure, so in practice more stack space may be allocated then used in principal; as long as there is plenty of stack this should have no specific impacts on code. From what I can tell, TPC-H results are in the neighborhood of the equivalent unpatched version, which is promising compared to the previous version. --- configure | 14 +- configure.ac | 25 +-- contrib/amcheck/verify_heapam.c | 10 +- contrib/bloom/bloom.h | 22 +- contrib/bloom/blutils.c | 4 +- contrib/bloom/blvacuum.c | 2 +- contrib/file_fdw/file_fdw.c | 4 +- contrib/pageinspect/rawpage.c | 2 +- contrib/pg_surgery/heap_surgery.c | 2 +- doc/src/sgml/config.sgml | 9 +- doc/src/sgml/monitoring.sgml | 2 +- doc/src/sgml/pgfreespacemap.sgml | 3 +- doc/src/sgml/ref/initdb.sgml | 16 ++ doc/src/sgml/ref/pg_resetwal.sgml | 19 ++ meson.build | 11 +- src/backend/access/common/reloptions.c | 33 ++- src/backend/access/common/syncscan.c | 2 +- src/backend/access/common/toast_internals.c | 2 +- src/backend/access/gin/gindatapage.c | 2 +- src/backend/access/gin/ginget.c | 6 +- src/backend/access/gin/ginvacuum.c | 2 +- src/backend/access/gist/gist.c | 2 +- src/backend/access/gist/gistvacuum.c | 6 +- src/backend/access/hash/hash.c | 4 +- src/backend/access/hash/hashinsert.c | 2 +- src/backend/access/hash/hashovfl.c | 8 +- src/backend/access/hash/hashpage.c | 4 +- src/backend/access/heap/heapam.c | 10 +- src/backend/access/heap/heapam_handler.c | 6 +- src/backend/access/heap/heaptoast.c | 2 +- src/backend/access/heap/hio.c | 2 +- src/backend/access/heap/pruneheap.c | 12 +- src/backend/access/heap/rewriteheap.c | 2 +- src/backend/access/heap/vacuumlazy.c | 12 +- src/backend/access/heap/visibilitymap.c | 6 +- src/backend/access/nbtree/nbtinsert.c | 2 +- src/backend/access/nbtree/nbtpage.c | 8 +- src/backend/access/nbtree/nbtree.c | 4 +- src/backend/access/nbtree/nbtxlog.c | 4 +- src/backend/access/spgist/spgdoinsert.c | 2 +- src/backend/access/spgist/spgscan.c | 4 +- src/backend/access/spgist/spgvacuum.c | 22 +- src/backend/access/transam/generic_xlog.c | 2 +- src/backend/access/transam/timeline.c | 2 +- src/backend/access/transam/xlog.c | 130 ++++++++++-- src/backend/access/transam/xloginsert.c | 6 +- src/backend/backup/basebackup.c | 13 +- src/backend/bootstrap/bootstrap.c | 14 +- src/backend/commands/async.c | 4 +- src/backend/nodes/tidbitmap.c | 4 +- src/backend/po/de.po | 2 +- src/backend/storage/buffer/bufmgr.c | 2 +- src/backend/storage/buffer/localbuf.c | 2 +- src/backend/storage/file/buffile.c | 4 +- src/backend/storage/file/fd.c | 4 +- src/backend/storage/freespace/freespace.c | 15 +- src/backend/storage/large_object/inv_api.c | 4 +- src/backend/storage/page/bufpage.c | 12 +- src/backend/storage/smgr/README | 52 +++++ src/backend/storage/smgr/md.c | 84 ++++---- src/backend/utils/init/miscinit.c | 6 +- src/backend/utils/misc/guc.c | 25 ++- src/backend/utils/misc/guc_tables.c | 39 +++- src/backend/utils/sort/logtape.c | 2 +- src/bin/initdb/initdb.c | 59 ++++-- src/bin/initdb/meson.build | 1 + src/bin/initdb/t/002_blocksize.pl | 24 +++ src/bin/pg_basebackup/pg_basebackup.c | 7 + src/bin/pg_basebackup/pg_receivewal.c | 4 + src/bin/pg_basebackup/streamutil.c | 59 ++++++ src/bin/pg_basebackup/streamutil.h | 3 + src/bin/pg_checksums/pg_checksums.c | 22 +- src/bin/pg_resetwal/pg_resetwal.c | 54 ++++- src/bin/pg_resetwal/t/002_corrupted.pl | 4 +- src/bin/pg_rewind/filemap.c | 1 + src/bin/pg_rewind/pg_rewind.c | 8 + src/bin/pg_upgrade/file.c | 17 +- src/bin/pg_upgrade/meson.build | 2 + src/bin/pg_upgrade/pg_upgrade.c | 3 + src/bin/pg_upgrade/pg_upgrade.h | 2 +- src/bin/pg_verifybackup/pg_verifybackup.c | 4 +- src/bin/pg_waldump/pg_waldump.c | 31 ++- src/bin/pg_waldump/t/001_basic.pl | 2 + src/bin/pg_waldump/t/002_save_fullpage.pl | 3 + src/common/Makefile | 1 + src/common/blocksize.c | 53 +++++ src/common/file_utils.c | 6 +- src/common/meson.build | 1 + src/include/access/gist_private.h | 2 +- src/include/access/hash.h | 4 +- src/include/access/heapam.h | 2 +- src/include/access/heaptoast.h | 20 +- src/include/access/htup_details.h | 16 +- src/include/access/itup.h | 6 +- src/include/access/nbtree.h | 207 +----------------- src/include/access/nbtree_int.h | 223 ++++++++++++++++++++ src/include/access/spgist_private.h | 10 +- src/include/access/xlog_internal.h | 1 + src/include/c.h | 6 +- src/include/common/blocksize.h | 119 +++++++++++ src/include/pg_config.h.in | 39 ++-- src/include/pg_config_manual.h | 8 + src/include/postgres.h | 1 + src/include/postgres_ext.h | 1 - src/include/storage/bufpage.h | 18 +- src/include/storage/checksum.h | 2 +- src/include/storage/checksum_impl.h | 13 +- src/include/storage/freespace.h | 1 + src/include/storage/large_object.h | 2 + src/include/storage/off.h | 4 +- src/include/utils/guc_tables.h | 1 + src/tools/msvc/Mkvcbuild.pm | 2 +- src/tools/msvc/Solution.pm | 2 +- src/tools/pgindent/typedefs.list | 1 - 114 files changed, 1277 insertions(+), 549 deletions(-) create mode 100644 src/bin/initdb/t/002_blocksize.pl create mode 100644 src/common/blocksize.c create mode 100644 src/include/access/nbtree_int.h create mode 100644 src/include/common/blocksize.h diff --git a/configure b/configure index d47e0f8b26..d8512697d9 100755 --- a/configure +++ b/configure @@ -3711,12 +3711,12 @@ fi case ${blocksize} in - 1) BLCKSZ=1024;; - 2) BLCKSZ=2048;; - 4) BLCKSZ=4096;; - 8) BLCKSZ=8192;; - 16) BLCKSZ=16384;; - 32) BLCKSZ=32768;; + 1) DEFAULT_BLOCK_SIZE=1024;; + 2) DEFAULT_BLOCK_SIZE=2048;; + 4) DEFAULT_BLOCK_SIZE=4096;; + 8) DEFAULT_BLOCK_SIZE=8192;; + 16) DEFAULT_BLOCK_SIZE=16384;; + 32) DEFAULT_BLOCK_SIZE=32768;; *) as_fn_error $? "Invalid block size. Allowed values are 1,2,4,8,16,32." "$LINENO" 5 esac { $as_echo "$as_me:${as_lineno-$LINENO}: result: ${blocksize}kB" >&5 @@ -3724,7 +3724,7 @@ $as_echo "${blocksize}kB" >&6; } cat >>confdefs.h <<_ACEOF -#define BLCKSZ ${BLCKSZ} +#define DEFAULT_BLOCK_SIZE ${DEFAULT_BLOCK_SIZE} _ACEOF diff --git a/configure.ac b/configure.ac index 440b08d113..0d6b8b3b07 100644 --- a/configure.ac +++ b/configure.ac @@ -258,28 +258,29 @@ PGAC_ARG_REQ(with, blocksize, [BLOCKSIZE], [set table block size in kB [8]], [blocksize=$withval], [blocksize=8]) case ${blocksize} in - 1) BLCKSZ=1024;; - 2) BLCKSZ=2048;; - 4) BLCKSZ=4096;; - 8) BLCKSZ=8192;; - 16) BLCKSZ=16384;; - 32) BLCKSZ=32768;; + 1) DEFAULT_BLOCK_SIZE=1024;; + 2) DEFAULT_BLOCK_SIZE=2048;; + 4) DEFAULT_BLOCK_SIZE=4096;; + 8) DEFAULT_BLOCK_SIZE=8192;; + 16) DEFAULT_BLOCK_SIZE=16384;; + 32) DEFAULT_BLOCK_SIZE=32768;; *) AC_MSG_ERROR([Invalid block size. Allowed values are 1,2,4,8,16,32.]) esac AC_MSG_RESULT([${blocksize}kB]) -AC_DEFINE_UNQUOTED([BLCKSZ], ${BLCKSZ}, [ +AC_DEFINE_UNQUOTED([DEFAULT_BLOCK_SIZE], ${DEFAULT_BLOCK_SIZE}, [ Size of a disk block --- this also limits the size of a tuple. You can set it bigger if you need bigger tuples (although TOAST should reduce the need to have large tuples, since fields can be spread across multiple tuples). - BLCKSZ must be a power of 2. The maximum possible value of BLCKSZ + DEFAULT_BLOCK_SIZE must be a power of 2. The maximum possible value of DEFAULT_BLOCK_SIZE is currently 2^15 (32768). This is determined by the 15-bit widths of the lp_off and lp_len fields in ItemIdData (see include/storage/itemid.h). - Changing BLCKSZ requires an initdb. + Changing DEFAULT_BLOCK_SIZE requires an initdb, however an individual cluster + can have its block size set. ]) # @@ -313,10 +314,10 @@ fi AC_DEFINE_UNQUOTED([RELSEG_SIZE], ${RELSEG_SIZE}, [ RELSEG_SIZE is the maximum number of blocks allowed in one disk file. - Thus, the maximum size of a single file is RELSEG_SIZE * BLCKSZ; + Thus, the maximum size of a single file is RELSEG_SIZE * cluster_block_size; relations bigger than that are divided into multiple files. - RELSEG_SIZE * BLCKSZ must be less than your OS' limit on file size. + RELSEG_SIZE * cluster_block_size must be less than your OS' limit on file size. This is often 2 GB or 4GB in a 32-bit operating system, unless you have large file support enabled. By default, we make the limit 1 GB to avoid any possible integer-overflow problems within the OS. @@ -350,7 +351,7 @@ esac AC_MSG_RESULT([${wal_blocksize}kB]) AC_DEFINE_UNQUOTED([XLOG_BLCKSZ], ${XLOG_BLCKSZ}, [ - Size of a WAL file block. This need have no particular relation to BLCKSZ. + Size of a WAL file block. This need have no particular relation to cluster_block_size. XLOG_BLCKSZ must be a power of 2, and if your system supports O_DIRECT I/O, XLOG_BLCKSZ must be a multiple of the alignment requirement for direct-I/O buffers, else direct I/O may fail. diff --git a/contrib/amcheck/verify_heapam.c b/contrib/amcheck/verify_heapam.c index 004aa26b7c..450496dd8a 100644 --- a/contrib/amcheck/verify_heapam.c +++ b/contrib/amcheck/verify_heapam.c @@ -403,11 +403,11 @@ verify_heapam(PG_FUNCTION_ARGS) for (ctx.blkno = first_block; ctx.blkno <= last_block; ctx.blkno++) { OffsetNumber maxoff; - OffsetNumber predecessor[MaxOffsetNumber]; - OffsetNumber successor[MaxOffsetNumber]; - bool lp_valid[MaxOffsetNumber]; - bool xmin_commit_status_ok[MaxOffsetNumber]; - XidCommitStatus xmin_commit_status[MaxOffsetNumber]; + OffsetNumber predecessor[MaxOffsetNumberLimit]; + OffsetNumber successor[MaxOffsetNumberLimit]; + bool lp_valid[MaxOffsetNumberLimit]; + bool xmin_commit_status_ok[MaxOffsetNumberLimit]; + XidCommitStatus xmin_commit_status[MaxOffsetNumberLimit]; CHECK_FOR_INTERRUPTS(); diff --git a/contrib/bloom/bloom.h b/contrib/bloom/bloom.h index d08fe1de26..e3f54ac02b 100644 --- a/contrib/bloom/bloom.h +++ b/contrib/bloom/bloom.h @@ -19,6 +19,7 @@ #include "access/xlog.h" #include "fmgr.h" #include "nodes/pathnodes.h" +#include "storage/bufpage.h" /* Support procedures numbers */ #define BLOOM_HASH_PROC 1 @@ -106,16 +107,9 @@ typedef struct BloomOptions * index key */ } BloomOptions; -/* - * FreeBlockNumberArray - array of block numbers sized so that metadata fill - * all space in metapage. - */ -typedef BlockNumber FreeBlockNumberArray[ - MAXALIGN_DOWN( - cluster_block_size - SizeOfPageHeaderData - MAXALIGN(sizeof(BloomPageOpaqueData)) - - MAXALIGN(sizeof(uint16) * 2 + sizeof(uint32) + sizeof(BloomOptions)) - ) / sizeof(BlockNumber) -]; +#define free_block_number_elems CalcFreeBlockNumberElems(cluster_block_size) +BlockSizeDecl(CalcFreeBlockNumberElems); + /* Metadata of bloom index */ typedef struct BloomMetaPageData @@ -124,14 +118,18 @@ typedef struct BloomMetaPageData uint16 nStart; uint16 nEnd; BloomOptions opts; - FreeBlockNumberArray notFullPage; + BlockNumber notFullPage[FLEXIBLE_ARRAY_MEMBER]; } BloomMetaPageData; +#define SizeOfBloomMetaPageData (offsetof(BloomMetaPageData,notFullPage) + sizeof(BlockNumber) * BloomMetaBlockN) + /* Magic number to distinguish bloom pages among anothers */ #define BLOOM_MAGICK_NUMBER (0xDBAC0DED) /* Number of blocks numbers fit in BloomMetaPageData */ -#define BloomMetaBlockN (sizeof(FreeBlockNumberArray) / sizeof(BlockNumber)) +#define BloomMetaBlockN (MAXALIGN_DOWN(cluster_block_size - SizeOfPageHeaderData - MAXALIGN(sizeof(BloomPageOpaqueData)) \ + - MAXALIGN(sizeof(uint16) * 2 + sizeof(uint32) + sizeof(BloomOptions)) \ + ) / sizeof(BlockNumber)) #define BloomPageGetMeta(page) ((BloomMetaPageData *) PageGetContents(page)) diff --git a/contrib/bloom/blutils.c b/contrib/bloom/blutils.c index 9d79319e90..89aa31ec73 100644 --- a/contrib/bloom/blutils.c +++ b/contrib/bloom/blutils.c @@ -430,10 +430,10 @@ BloomFillMetapage(Relation index, Page metaPage) */ BloomInitPage(metaPage, BLOOM_META); metadata = BloomPageGetMeta(metaPage); - memset(metadata, 0, sizeof(BloomMetaPageData)); + memset(metadata, 0, SizeOfBloomMetaPageData); metadata->magickNumber = BLOOM_MAGICK_NUMBER; metadata->opts = *opts; - ((PageHeader) metaPage)->pd_lower += sizeof(BloomMetaPageData); + ((PageHeader) metaPage)->pd_lower += SizeOfBloomMetaPageData; /* If this fails, probably FreeBlockNumberArray size calc is wrong: */ Assert(((PageHeader) metaPage)->pd_lower <= ((PageHeader) metaPage)->pd_upper); diff --git a/contrib/bloom/blvacuum.c b/contrib/bloom/blvacuum.c index 2340d49e00..373227af98 100644 --- a/contrib/bloom/blvacuum.c +++ b/contrib/bloom/blvacuum.c @@ -37,7 +37,7 @@ blbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, Relation index = info->index; BlockNumber blkno, npages; - FreeBlockNumberArray notFullPage; + BlockNumber notFullPage[CalcFreeBlockNumberElems(MAX_BLOCK_SIZE)]; int countPage = 0; BloomState state; Buffer buffer; diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index c265b6e309..13a3a73ebd 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -840,7 +840,7 @@ fileAnalyzeForeignTable(Relation relation, * Convert size to pages. Must return at least 1 so that we can tell * later on that pg_class.relpages is not default. */ - *totalpages = (stat_buf.st_size + (cluster_block_size - 1)) / cluster_block_size; + *totalpages = (stat_buf.st_size + (cluster_block_size - 1)) >> cluster_block_bits; if (*totalpages < 1) *totalpages = 1; @@ -1015,7 +1015,7 @@ estimate_size(PlannerInfo *root, RelOptInfo *baserel, /* * Convert size to pages for use in I/O cost estimate later. */ - pages = (stat_buf.st_size + (cluster_block_size - 1)) / cluster_block_size; + pages = (stat_buf.st_size + (cluster_block_size - 1)) >> cluster_block_bits; if (pages < 1) pages = 1; fdw_private->pages = pages; diff --git a/contrib/pageinspect/rawpage.c b/contrib/pageinspect/rawpage.c index 2527501078..bdeeb027d3 100644 --- a/contrib/pageinspect/rawpage.c +++ b/contrib/pageinspect/rawpage.c @@ -357,7 +357,7 @@ page_checksum_internal(PG_FUNCTION_ARGS, enum pageinspect_version ext_version) if (PageIsNew(page)) PG_RETURN_NULL(); - PG_RETURN_INT16(pg_checksum_page((char *) page, blkno)); + PG_RETURN_INT16(pg_checksum_page((char *) page, blkno, cluster_block_size)); } Datum diff --git a/contrib/pg_surgery/heap_surgery.c b/contrib/pg_surgery/heap_surgery.c index 88a40ab7d3..58c4819675 100644 --- a/contrib/pg_surgery/heap_surgery.c +++ b/contrib/pg_surgery/heap_surgery.c @@ -89,7 +89,7 @@ heap_force_common(FunctionCallInfo fcinfo, HeapTupleForceOption heap_force_opt) Relation rel; OffsetNumber curr_start_ptr, next_start_ptr; - bool include_this_tid[MaxHeapTuplesPerPage]; + bool include_this_tid[MaxHeapTuplesPerPageLimit]; if (RecoveryInProgress()) ereport(ERROR, diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 2274bc0ffb..19510505b5 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -10625,12 +10625,13 @@ dynamic_library_path = 'C:\tools\postgresql;H:\my_project\lib;$libdir' - Reports the size of a disk block. It is determined by the value - of cluster_block_size when building the server. The default - value is 8192 bytes. The meaning of some configuration + Reports the size of a disk block for this cluster. + The default value is 8192 bytes. The meaning of some configuration variables (such as ) is influenced by block_size. See for information. + linkend="runtime-config-resource"/> for information on GUC configuration + or for more information on + selecting a cluster's block size. diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index c655c70e38..b80b43780e 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -2692,7 +2692,7 @@ description | Waiting for a newly initialized WAL file to reach durable storage Relation data reads, writes, and extends are done in - block_size units, derived from the build-time + block_size units, derived from the cluster init-time parameter cluster_block_size, which is 8192 by default. diff --git a/doc/src/sgml/pgfreespacemap.sgml b/doc/src/sgml/pgfreespacemap.sgml index 0da94e5f63..1777c044ab 100644 --- a/doc/src/sgml/pgfreespacemap.sgml +++ b/doc/src/sgml/pgfreespacemap.sgml @@ -60,7 +60,8 @@ The values stored in the free space map are not exact. They're rounded - to precision of 1/256th of cluster_block_size (32 bytes with default cluster_block_size), and + to precision of 1/256th of cluster_block_size (32 bytes + with default cluster_block_size), and they're not kept fully up-to-date as tuples are inserted and updated. diff --git a/doc/src/sgml/ref/initdb.sgml b/doc/src/sgml/ref/initdb.sgml index 22f1011781..ed8b20a0aa 100644 --- a/doc/src/sgml/ref/initdb.sgml +++ b/doc/src/sgml/ref/initdb.sgml @@ -277,6 +277,22 @@ PostgreSQL documentation + + + + + + Sets the cluster's database block size + to blocksize kilobytes. This should be one + of 1, 2, 4, 8, 16, or 32. Many database limits are defined relative + to this parameter. If unspecified, defaults to 8. + + + This used to be the compile-time parameter cluster_block_size. + + + + diff --git a/doc/src/sgml/ref/pg_resetwal.sgml b/doc/src/sgml/ref/pg_resetwal.sgml index fd539f5604..54dd73bb12 100644 --- a/doc/src/sgml/ref/pg_resetwal.sgml +++ b/doc/src/sgml/ref/pg_resetwal.sgml @@ -136,6 +136,25 @@ PostgreSQL documentation + + + + + + Sets the cluster's database block size + to blocksize kilobytes. This should be one + of 1, 2, 4, 8, 16, or 32. Many database limits are defined relative + to this parameter. + + + This must be the same value selected + at initdb time, or this will result in database + corruption. This option should only be provided + when pg_control is corrupted or missing. + + + + diff --git a/meson.build b/meson.build index 5422885b0a..9794be4793 100644 --- a/meson.build +++ b/meson.build @@ -444,14 +444,15 @@ else segsize = (get_option('segsize') * 1024 * 1024 * 1024) / blocksize endif -cdata.set('BLCKSZ', blocksize, description: -'''Size of a disk block --- this also limits the size of a tuple. You can set +cdata.set('DEFAULT_BLOCK_SIZE', blocksize, description: +'''Default size of a disk block --- this also limits the size of a tuple. You can set it bigger if you need bigger tuples (although TOAST should reduce the need to have large tuples, since fields can be spread across multiple tuples). - BLCKSZ must be a power of 2. The maximum possible value of BLCKSZ is + DEFAULT_BLOCK_SIZE must be a power of 2. The maximum possible value of DEFAULT_BLOCK_SIZE is currently 2^15 (32768). This is determined by the 15-bit widths of the lp_off and lp_len fields in ItemIdData (see include/storage/itemid.h). - Changing BLCKSZ requires an initdb.''') + Changing DEFAULT_BLOCK_SIZE requires an initdb, however individual clusters + can have different values for block size (cluster_block_size)''') cdata.set('XLOG_BLCKSZ', get_option('wal_blocksize').to_int() * 1024) cdata.set('RELSEG_SIZE', segsize) @@ -3336,7 +3337,7 @@ if meson.version().version_compare('>=0.57') summary( { - 'data block size': '@0@ kB'.format(cdata.get('BLCKSZ') / 1024), + 'data block size': '@0@ kB'.format(cdata.get('DEFAULT_BLOCK_SIZE') / 1024), 'WAL block size': '@0@ kB'.format(cdata.get('XLOG_BLCKSZ') / 1024), 'segment size': get_option('segsize_blocks') != 0 ? '@0@ blocks'.format(cdata.get('RELSEG_SIZE')) : diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c index 52d4ee6834..9d8a16f6ad 100644 --- a/src/backend/access/common/reloptions.c +++ b/src/backend/access/common/reloptions.c @@ -28,6 +28,7 @@ #include "commands/defrem.h" #include "commands/tablespace.h" #include "commands/view.h" +#include "common/blocksize.h" #include "nodes/makefuncs.h" #include "postmaster/postmaster.h" #include "utils/array.h" @@ -328,7 +329,8 @@ static relopt_int intRelOpts[] = RELOPT_KIND_HEAP, ShareUpdateExclusiveLock }, - cluster_toast_tuple_target, 128, toast_tuple_target_main + /* NOTE: these limits are dynamically updated */ + 0, 0, 0 }, { { @@ -563,6 +565,8 @@ static void initialize_reloptions(void); static void parse_one_reloption(relopt_value *option, char *text_str, int text_len, bool validate); +static void update_dynamic_reloptions(void); + /* * Get the length of a string reloption (either default or the user-defined * value). This is used for allocation purposes when building a set of @@ -572,6 +576,26 @@ static void parse_one_reloption(relopt_value *option, char *text_str, ((option).isset ? strlen((option).values.string_val) : \ ((relopt_string *) (option).gen)->default_len) +/* + * handle adjustments to the config table based on dynamic parameters' limits + */ + +static void +update_dynamic_reloptions(void) +{ + int i; + + for (i = 0; intRelOpts[i].gen.name; i++) + { + if (strcmp("toast_tuple_target", intRelOpts[i].gen.name) == 0) + { + intRelOpts[i].min = 128; + intRelOpts[i].default_val = cluster_toast_tuple_target; + intRelOpts[i].max = toast_tuple_target_main; + } + } +} + /* * initialize_reloptions * initialization routine, must be called before parsing @@ -584,6 +608,13 @@ initialize_reloptions(void) int i; int j; + /* + * Set the dynamic limits based on block size; if we get multiple can make + * more sophisticated. + */ + + update_dynamic_reloptions(); + j = 0; for (i = 0; boolRelOpts[i].gen.name; i++) { diff --git a/src/backend/access/common/syncscan.c b/src/backend/access/common/syncscan.c index 3aa54f28d4..1d8e4b35a9 100644 --- a/src/backend/access/common/syncscan.c +++ b/src/backend/access/common/syncscan.c @@ -80,7 +80,7 @@ bool trace_syncscan = false; * the buffer cache anyway, and on the other hand the page is most likely * still in the OS cache. */ -#define SYNC_SCAN_REPORT_INTERVAL (128 * 1024 / cluster_block_size) +#define SYNC_SCAN_REPORT_INTERVAL (128 * 1024 >> cluster_block_bits) /* diff --git a/src/backend/access/common/toast_internals.c b/src/backend/access/common/toast_internals.c index 8745f87be6..7c87b938fc 100644 --- a/src/backend/access/common/toast_internals.c +++ b/src/backend/access/common/toast_internals.c @@ -133,7 +133,7 @@ toast_save_datum(Relation rel, Datum value, { struct varlena hdr; /* this is to make the union big enough for a chunk: */ - char data[cluster_toast_max_chunk_size + VARHDRSZ]; + char data[TOAST_MAX_CHUNK_SIZE_LIMIT + VARHDRSZ]; /* ensure union is aligned well enough: */ int32 align_it; } chunk_data; diff --git a/src/backend/access/gin/gindatapage.c b/src/backend/access/gin/gindatapage.c index 9bef09651e..9c42501a8d 100644 --- a/src/backend/access/gin/gindatapage.c +++ b/src/backend/access/gin/gindatapage.c @@ -1259,7 +1259,7 @@ dataSplitPageInternal(GinBtree btree, Buffer origbuf, Page lpage; Page rpage; OffsetNumber separator; - PostingItem allitems[(cluster_block_size / sizeof(PostingItem)) + 1]; + PostingItem allitems[(MAX_BLOCK_SIZE / sizeof(PostingItem)) + 1]; lpage = PageGetTempPage(oldpage); rpage = PageGetTempPage(oldpage); diff --git a/src/backend/access/gin/ginget.c b/src/backend/access/gin/ginget.c index 022f71e25c..59aa2a73e4 100644 --- a/src/backend/access/gin/ginget.c +++ b/src/backend/access/gin/ginget.c @@ -1632,9 +1632,9 @@ collectMatchesForHeapRow(IndexScanDesc scan, pendingPosition *pos) */ for (;;) { - Datum datum[cluster_block_size / sizeof(IndexTupleData)]; - GinNullCategory category[cluster_block_size / sizeof(IndexTupleData)]; - bool datumExtracted[cluster_block_size / sizeof(IndexTupleData)]; + Datum datum[MAX_BLOCK_SIZE / sizeof(IndexTupleData)]; + GinNullCategory category[MAX_BLOCK_SIZE / sizeof(IndexTupleData)]; + bool datumExtracted[MAX_BLOCK_SIZE / sizeof(IndexTupleData)]; Assert(pos->lastOffset > pos->firstOffset); memset(datumExtracted + pos->firstOffset - 1, 0, diff --git a/src/backend/access/gin/ginvacuum.c b/src/backend/access/gin/ginvacuum.c index 628d0f3a40..c7198d85c9 100644 --- a/src/backend/access/gin/ginvacuum.c +++ b/src/backend/access/gin/ginvacuum.c @@ -569,7 +569,7 @@ ginbulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, BlockNumber blkno = GIN_ROOT_BLKNO; GinVacuumState gvs; Buffer buffer; - BlockNumber rootOfPostingTree[cluster_block_size / (sizeof(IndexTupleData) + sizeof(ItemId))]; + BlockNumber rootOfPostingTree[MAX_BLOCK_SIZE / (sizeof(IndexTupleData) + sizeof(ItemId))]; uint32 nRoot; gvs.tmpCxt = AllocSetContextCreate(CurrentMemoryContext, diff --git a/src/backend/access/gist/gist.c b/src/backend/access/gist/gist.c index b1f19f6a8e..4fbfd986a1 100644 --- a/src/backend/access/gist/gist.c +++ b/src/backend/access/gist/gist.c @@ -1642,7 +1642,7 @@ freeGISTstate(GISTSTATE *giststate) static void gistprunepage(Relation rel, Page page, Buffer buffer, Relation heapRel) { - OffsetNumber deletable[MaxIndexTuplesPerPage]; + OffsetNumber deletable[MaxIndexTuplesPerPageLimit]; int ndeletable = 0; OffsetNumber offnum, maxoff; diff --git a/src/backend/access/gist/gistvacuum.c b/src/backend/access/gist/gistvacuum.c index 3f60d3274d..53a29e55b3 100644 --- a/src/backend/access/gist/gistvacuum.c +++ b/src/backend/access/gist/gistvacuum.c @@ -309,7 +309,7 @@ restart: } else if (GistPageIsLeaf(page)) { - OffsetNumber todelete[MaxOffsetNumber]; + OffsetNumber todelete[MaxOffsetNumberLimit]; int ntodelete = 0; int nremain; GISTPageOpaque opaque = GistPageGetOpaque(page); @@ -476,8 +476,8 @@ gistvacuum_delete_empty_pages(IndexVacuumInfo *info, GistVacState *vstate) Page page; OffsetNumber off, maxoff; - OffsetNumber todelete[MaxOffsetNumber]; - BlockNumber leafs_to_delete[MaxOffsetNumber]; + OffsetNumber todelete[MaxOffsetNumberLimit]; + BlockNumber leafs_to_delete[MaxOffsetNumberLimit]; int ntodelete; int deleted; diff --git a/src/backend/access/hash/hash.c b/src/backend/access/hash/hash.c index ea78dc81c2..452119220d 100644 --- a/src/backend/access/hash/hash.c +++ b/src/backend/access/hash/hash.c @@ -152,7 +152,7 @@ hashbuild(Relation heap, Relation index, IndexInfo *indexInfo) * one page. Also, "initial index size" accounting does not include the * metapage, nor the first bitmap page. */ - sort_threshold = (maintenance_work_mem * 1024L) / cluster_block_size; + sort_threshold = (maintenance_work_mem * 1024L) >> cluster_block_bits; if (index->rd_rel->relpersistence != RELPERSISTENCE_TEMP) sort_threshold = Min(sort_threshold, NBuffers); else @@ -709,7 +709,7 @@ hashbucketcleanup(Relation rel, Bucket cur_bucket, Buffer bucket_buf, OffsetNumber maxoffno; Buffer next_buf; Page page; - OffsetNumber deletable[MaxOffsetNumber]; + OffsetNumber deletable[MaxOffsetNumberLimit]; int ndeletable = 0; bool retain_pin = false; bool clear_dead_marking = false; diff --git a/src/backend/access/hash/hashinsert.c b/src/backend/access/hash/hashinsert.c index 22656b24e2..48d986272a 100644 --- a/src/backend/access/hash/hashinsert.c +++ b/src/backend/access/hash/hashinsert.c @@ -371,7 +371,7 @@ _hash_pgaddmultitup(Relation rel, Buffer buf, IndexTuple *itups, static void _hash_vacuum_one_page(Relation rel, Relation hrel, Buffer metabuf, Buffer buf) { - OffsetNumber deletable[MaxOffsetNumber]; + OffsetNumber deletable[MaxOffsetNumberLimit]; int ndeletable = 0; OffsetNumber offnum, maxoff; diff --git a/src/backend/access/hash/hashovfl.c b/src/backend/access/hash/hashovfl.c index 39bb2cb9f6..dd414c557e 100644 --- a/src/backend/access/hash/hashovfl.c +++ b/src/backend/access/hash/hashovfl.c @@ -866,10 +866,10 @@ _hash_squeezebucket(Relation rel, { OffsetNumber roffnum; OffsetNumber maxroffnum; - OffsetNumber deletable[MaxOffsetNumber]; - IndexTuple itups[MaxIndexTuplesPerPage]; - Size tups_size[MaxIndexTuplesPerPage]; - OffsetNumber itup_offsets[MaxIndexTuplesPerPage]; + OffsetNumber deletable[MaxOffsetNumberLimit]; + IndexTuple itups[MaxIndexTuplesPerPageLimit]; + Size tups_size[MaxIndexTuplesPerPageLimit]; + OffsetNumber itup_offsets[MaxIndexTuplesPerPageLimit]; uint16 ndeletable = 0; uint16 nitups = 0; Size all_tups_size = 0; diff --git a/src/backend/access/hash/hashpage.c b/src/backend/access/hash/hashpage.c index 2a77d13feb..1cc99ae27a 100644 --- a/src/backend/access/hash/hashpage.c +++ b/src/backend/access/hash/hashpage.c @@ -1087,8 +1087,8 @@ _hash_splitbucket(Relation rel, Page npage; HashPageOpaque oopaque; HashPageOpaque nopaque; - OffsetNumber itup_offsets[MaxIndexTuplesPerPage]; - IndexTuple itups[MaxIndexTuplesPerPage]; + OffsetNumber itup_offsets[MaxIndexTuplesPerPageLimit]; + IndexTuple itups[MaxIndexTuplesPerPageLimit]; Size all_tups_size = 0; int i; uint16 nitups = 0; diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index ef3f3e0bdf..8c16ae0ea2 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -6741,8 +6741,8 @@ heap_freeze_execute_prepared(Relation rel, Buffer buffer, /* Now WAL-log freezing if necessary */ if (RelationNeedsWAL(rel)) { - xl_heap_freeze_plan plans[MaxHeapTuplesPerPage]; - OffsetNumber offsets[MaxHeapTuplesPerPage]; + xl_heap_freeze_plan plans[MaxHeapTuplesPerPageLimit]; + OffsetNumber offsets[MaxHeapTuplesPerPageLimit]; int nplans; xl_heap_freeze_page xlrec; XLogRecPtr recptr; @@ -9222,7 +9222,7 @@ heap_xlog_insert(XLogReaderState *record) union { HeapTupleHeaderData hdr; - char data[MaxHeapTupleSize]; + char data[MaxHeapTupleSizeLimit]; } tbuf; HeapTupleHeader htup; xl_heap_header xlhdr; @@ -9344,7 +9344,7 @@ heap_xlog_multi_insert(XLogReaderState *record) union { HeapTupleHeaderData hdr; - char data[MaxHeapTupleSize]; + char data[MaxHeapTupleSizeLimit]; } tbuf; HeapTupleHeader htup; uint32 newlen; @@ -9501,7 +9501,7 @@ heap_xlog_update(XLogReaderState *record, bool hot_update) union { HeapTupleHeaderData hdr; - char data[MaxHeapTupleSize]; + char data[MaxHeapTupleSizeLimit]; } tbuf; xl_heap_header xlhdr; uint32 newlen; diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 6857b46203..4cbfca1996 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -1195,7 +1195,7 @@ heapam_index_build_range_scan(Relation heapRelation, TransactionId OldestXmin; BlockNumber previous_blkno = InvalidBlockNumber; BlockNumber root_blkno = InvalidBlockNumber; - OffsetNumber root_offsets[MaxHeapTuplesPerPage]; + OffsetNumber root_offsets[MaxHeapTuplesPerPageLimit]; /* * sanity checks @@ -1758,8 +1758,8 @@ heapam_index_validate_scan(Relation heapRelation, EState *estate; ExprContext *econtext; BlockNumber root_blkno = InvalidBlockNumber; - OffsetNumber root_offsets[MaxHeapTuplesPerPage]; - bool in_index[MaxHeapTuplesPerPage]; + OffsetNumber root_offsets[MaxHeapTuplesPerPageLimit]; + bool in_index[MaxHeapTuplesPerPageLimit]; BlockNumber previous_blkno = InvalidBlockNumber; /* state variables for the merge */ diff --git a/src/backend/access/heap/heaptoast.c b/src/backend/access/heap/heaptoast.c index 98b0a701e5..6c2d1c9dc1 100644 --- a/src/backend/access/heap/heaptoast.c +++ b/src/backend/access/heap/heaptoast.c @@ -770,7 +770,7 @@ heap_fetch_toast_slice(Relation toastrel, Oid valueid, int32 attrsize, chcpyend = (sliceoffset + slicelength - 1) % cluster_toast_max_chunk_size; memcpy(VARDATA(result) + - (curchunk * cluster_toast_max_chunk_size - sliceoffset) + chcpystrt, + ((int32)(curchunk * cluster_toast_max_chunk_size) - sliceoffset) + chcpystrt, chunkdata + chcpystrt, (chcpyend - chcpystrt) + 1); diff --git a/src/backend/access/heap/hio.c b/src/backend/access/heap/hio.c index caa62708aa..4f2018410a 100644 --- a/src/backend/access/heap/hio.c +++ b/src/backend/access/heap/hio.c @@ -534,7 +534,7 @@ RelationGetBufferForTuple(Relation relation, Size len, ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("row is too big: size %zu, maximum size %zu", - len, MaxHeapTupleSize))); + len, (Size)MaxHeapTupleSize))); /* Compute desired extra freespace due to fillfactor option */ saveFreeSpace = RelationGetTargetPageFreeSpace(relation, diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index 8b8b9c4d79..deba4d8ffd 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -54,9 +54,9 @@ typedef struct int ndead; int nunused; /* arrays that accumulate indexes of items to be changed */ - OffsetNumber redirected[MaxHeapTuplesPerPage * 2]; - OffsetNumber nowdead[MaxHeapTuplesPerPage]; - OffsetNumber nowunused[MaxHeapTuplesPerPage]; + OffsetNumber redirected[MaxHeapTuplesPerPageLimit * 2]; + OffsetNumber nowdead[MaxHeapTuplesPerPageLimit]; + OffsetNumber nowunused[MaxHeapTuplesPerPageLimit]; /* * marked[i] is true if item i is entered in one of the above arrays. @@ -64,7 +64,7 @@ typedef struct * This needs to be MaxHeapTuplesPerPage + 1 long as FirstOffsetNumber is * 1. Otherwise every access would need to subtract 1. */ - bool marked[MaxHeapTuplesPerPage + 1]; + bool marked[MaxHeapTuplesPerPageLimit + 1]; /* * Tuple visibility is only computed once for each tuple, for correctness @@ -74,7 +74,7 @@ typedef struct * * Same indexing as ->marked. */ - int8 htsv[MaxHeapTuplesPerPage + 1]; + int8 htsv[MaxHeapTuplesPerPageLimit + 1]; } PruneState; /* Local functions */ @@ -598,7 +598,7 @@ heap_prune_chain(Buffer buffer, OffsetNumber rootoffnum, PruneState *prstate) OffsetNumber latestdead = InvalidOffsetNumber, maxoff = PageGetMaxOffsetNumber(dp), offnum; - OffsetNumber chainitems[MaxHeapTuplesPerPage]; + OffsetNumber chainitems[MaxHeapTuplesPerPageLimit]; int nchain = 0, i; diff --git a/src/backend/access/heap/rewriteheap.c b/src/backend/access/heap/rewriteheap.c index 3402625d6d..9f9c90d6ea 100644 --- a/src/backend/access/heap/rewriteheap.c +++ b/src/backend/access/heap/rewriteheap.c @@ -657,7 +657,7 @@ raw_heap_insert(RewriteState state, HeapTuple tup) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("row is too big: size %zu, maximum size %zu", - len, MaxHeapTupleSize))); + len, (Size)MaxHeapTupleSize))); /* Compute desired extra freespace due to fillfactor option */ saveFreeSpace = RelationGetTargetPageFreeSpace(state->rs_new_rel, diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index d6f46ad60e..0bf1d13896 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -98,7 +98,7 @@ * (Note that this is deliberately kept to a power-of-two, usually 2^19.) */ #define failsafe_every_pages \ - ((BlockNumber) (((uint64) 4 * 1024 * 1024 * 1024) / cluster_block_size)) + ((BlockNumber) (((uint64) 4 * 1024 * 1024 * 1024) >> cluster_block_bits)) /* * When a table has no indexes, vacuum the FSM after every 8GB, approximately @@ -107,7 +107,7 @@ * and we vacuum FSM after each index/heap cleaning pass. */ #define vacuum_fsm_every_pages \ - ((BlockNumber) (((uint64) 8 * 1024 * 1024 * 1024) / cluster_block_size)) + ((BlockNumber) (((uint64) 8 * 1024 * 1024 * 1024) >> cluster_block_bits)) /* * Before we consider skipping a page that's marked as clean in @@ -1552,8 +1552,8 @@ lazy_scan_prune(LVRelState *vacrel, int nnewlpdead; HeapPageFreeze pagefrz; int64 fpi_before = pgWalUsage.wal_fpi; - OffsetNumber deadoffsets[MaxHeapTuplesPerPage]; - HeapTupleFreeze frozen[MaxHeapTuplesPerPage]; + OffsetNumber deadoffsets[MaxHeapTuplesPerPageLimit]; + HeapTupleFreeze frozen[MaxHeapTuplesPerPageLimit]; Assert(BufferGetBlockNumber(buf) == blkno); @@ -1972,7 +1972,7 @@ lazy_scan_noprune(LVRelState *vacrel, HeapTupleHeader tupleheader; TransactionId NoFreezePageRelfrozenXid = vacrel->NewRelfrozenXid; MultiXactId NoFreezePageRelminMxid = vacrel->NewRelminMxid; - OffsetNumber deadoffsets[MaxHeapTuplesPerPage]; + OffsetNumber deadoffsets[MaxHeapTuplesPerPageLimit]; Assert(BufferGetBlockNumber(buf) == blkno); @@ -2526,7 +2526,7 @@ lazy_vacuum_heap_page(LVRelState *vacrel, BlockNumber blkno, Buffer buffer, { VacDeadItems *dead_items = vacrel->dead_items; Page page = BufferGetPage(buffer); - OffsetNumber unused[MaxHeapTuplesPerPage]; + OffsetNumber unused[MaxHeapTuplesPerPageLimit]; int nunused = 0; TransactionId visibility_cutoff_xid; bool all_frozen; diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c index 833774c164..ab4d38fbe6 100644 --- a/src/backend/access/heap/visibilitymap.c +++ b/src/backend/access/heap/visibilitymap.c @@ -111,7 +111,7 @@ #define HEAPBLOCKS_PER_BYTE (BITS_PER_BYTE / BITS_PER_HEAPBLOCK) /* Number of heap blocks we can represent in one visibility map page. */ -#define HEAPBLOCKS_PER_PAGE (MAPSIZE * HEAPBLOCKS_PER_BYTE) +#define HEAPBLOCKS_PER_PAGE (MAPSIZE << BITS_PER_HEAPBLOCK) /* Mapping from heap block number to the right bit in the visibility map */ #define HEAPBLK_TO_MAPBLOCK(x) ((x) / HEAPBLOCKS_PER_PAGE) @@ -414,8 +414,8 @@ visibilitymap_count(Relation rel, BlockNumber *all_visible, BlockNumber *all_fro */ map = (uint64 *) PageGetContents(BufferGetPage(mapBuffer)); - StaticAssertStmt(MAPSIZE % sizeof(uint64) == 0, - "unsupported MAPSIZE"); + Assert(MAPSIZE % sizeof(uint64) == 0); + if (all_frozen == NULL) { for (i = 0; i < MAPSIZE / sizeof(uint64); i++) diff --git a/src/backend/access/nbtree/nbtinsert.c b/src/backend/access/nbtree/nbtinsert.c index d33f814a93..eed23d35fd 100644 --- a/src/backend/access/nbtree/nbtinsert.c +++ b/src/backend/access/nbtree/nbtinsert.c @@ -2685,7 +2685,7 @@ _bt_delete_or_dedup_one_page(Relation rel, Relation heapRel, bool simpleonly, bool checkingunique, bool uniquedup, bool indexUnchanged) { - OffsetNumber deletable[MaxIndexTuplesPerPage]; + OffsetNumber deletable[MaxIndexTuplesPerPageLimit]; int ndeletable = 0; OffsetNumber offnum, minoff, diff --git a/src/backend/access/nbtree/nbtpage.c b/src/backend/access/nbtree/nbtpage.c index 58dcdf6de4..fb6985b4cf 100644 --- a/src/backend/access/nbtree/nbtpage.c +++ b/src/backend/access/nbtree/nbtpage.c @@ -1160,7 +1160,7 @@ _bt_delitems_vacuum(Relation rel, Buffer buf, bool needswal = RelationNeedsWAL(rel); char *updatedbuf = NULL; Size updatedbuflen = 0; - OffsetNumber updatedoffsets[MaxIndexTuplesPerPage]; + OffsetNumber updatedoffsets[MaxIndexTuplesPerPageLimit]; /* Shouldn't be called unless there's something to do */ Assert(ndeletable > 0 || nupdatable > 0); @@ -1291,7 +1291,7 @@ _bt_delitems_delete(Relation rel, Buffer buf, bool needswal = RelationNeedsWAL(rel); char *updatedbuf = NULL; Size updatedbuflen = 0; - OffsetNumber updatedoffsets[MaxIndexTuplesPerPage]; + OffsetNumber updatedoffsets[MaxIndexTuplesPerPageLimit]; /* Shouldn't be called unless there's something to do */ Assert(ndeletable > 0 || nupdatable > 0); @@ -1524,8 +1524,8 @@ _bt_delitems_delete_check(Relation rel, Buffer buf, Relation heapRel, OffsetNumber postingidxoffnum = InvalidOffsetNumber; int ndeletable = 0, nupdatable = 0; - OffsetNumber deletable[MaxIndexTuplesPerPage]; - BTVacuumPosting updatable[MaxIndexTuplesPerPage]; + OffsetNumber deletable[MaxIndexTuplesPerPageLimit]; + BTVacuumPosting updatable[MaxIndexTuplesPerPageLimit]; /* Use tableam interface to determine which tuples to delete first */ snapshotConflictHorizon = table_index_delete_tuples(heapRel, delstate); diff --git a/src/backend/access/nbtree/nbtree.c b/src/backend/access/nbtree/nbtree.c index cb2757f68d..749a6bed9b 100644 --- a/src/backend/access/nbtree/nbtree.c +++ b/src/backend/access/nbtree/nbtree.c @@ -1156,9 +1156,9 @@ backtrack: } else if (P_ISLEAF(opaque)) { - OffsetNumber deletable[MaxIndexTuplesPerPage]; + OffsetNumber deletable[MaxIndexTuplesPerPageLimit]; int ndeletable; - BTVacuumPosting updatable[MaxIndexTuplesPerPage]; + BTVacuumPosting updatable[MaxIndexTuplesPerPageLimit]; int nupdatable; OffsetNumber offnum, minoff, diff --git a/src/backend/access/nbtree/nbtxlog.c b/src/backend/access/nbtree/nbtxlog.c index c87e46ed66..7035665eaf 100644 --- a/src/backend/access/nbtree/nbtxlog.c +++ b/src/backend/access/nbtree/nbtxlog.c @@ -40,8 +40,8 @@ _bt_restore_page(Page page, char *from, int len) IndexTupleData itupdata; Size itemsz; char *end = from + len; - Item items[MaxIndexTuplesPerPage]; - uint16 itemsizes[MaxIndexTuplesPerPage]; + Item items[MaxIndexTuplesPerPageLimit]; + uint16 itemsizes[MaxIndexTuplesPerPageLimit]; int i; int nitems; diff --git a/src/backend/access/spgist/spgdoinsert.c b/src/backend/access/spgist/spgdoinsert.c index d8c2752e85..02c5b31907 100644 --- a/src/backend/access/spgist/spgdoinsert.c +++ b/src/backend/access/spgist/spgdoinsert.c @@ -135,7 +135,7 @@ spgPageIndexMultiDelete(SpGistState *state, Page page, BlockNumber blkno, OffsetNumber offnum) { OffsetNumber firstItem; - OffsetNumber sortednos[MaxIndexTuplesPerPage]; + OffsetNumber sortednos[MaxIndexTuplesPerPageLimit]; SpGistDeadTuple tuple = NULL; int i; diff --git a/src/backend/access/spgist/spgscan.c b/src/backend/access/spgist/spgscan.c index cbfaf0c00a..8686916846 100644 --- a/src/backend/access/spgist/spgscan.c +++ b/src/backend/access/spgist/spgscan.c @@ -755,8 +755,8 @@ spgGetNextQueueItem(SpGistScanOpaque so) enum SpGistSpecialOffsetNumbers { SpGistBreakOffsetNumber = InvalidOffsetNumber, - SpGistRedirectOffsetNumber = MaxOffsetNumber + 1, - SpGistErrorOffsetNumber = MaxOffsetNumber + 2 + SpGistRedirectOffsetNumber = MaxOffsetNumberLimit + 1, + SpGistErrorOffsetNumber = MaxOffsetNumberLimit + 2 }; static OffsetNumber diff --git a/src/backend/access/spgist/spgvacuum.c b/src/backend/access/spgist/spgvacuum.c index 8a5b540c80..f561ff6e2b 100644 --- a/src/backend/access/spgist/spgvacuum.c +++ b/src/backend/access/spgist/spgvacuum.c @@ -128,14 +128,14 @@ vacuumLeafPage(spgBulkDeleteState *bds, Relation index, Buffer buffer, { Page page = BufferGetPage(buffer); spgxlogVacuumLeaf xlrec; - OffsetNumber toDead[MaxIndexTuplesPerPage]; - OffsetNumber toPlaceholder[MaxIndexTuplesPerPage]; - OffsetNumber moveSrc[MaxIndexTuplesPerPage]; - OffsetNumber moveDest[MaxIndexTuplesPerPage]; - OffsetNumber chainSrc[MaxIndexTuplesPerPage]; - OffsetNumber chainDest[MaxIndexTuplesPerPage]; - OffsetNumber predecessor[MaxIndexTuplesPerPage + 1]; - bool deletable[MaxIndexTuplesPerPage + 1]; + OffsetNumber toDead[MaxIndexTuplesPerPageLimit]; + OffsetNumber toPlaceholder[MaxIndexTuplesPerPageLimit]; + OffsetNumber moveSrc[MaxIndexTuplesPerPageLimit]; + OffsetNumber moveDest[MaxIndexTuplesPerPageLimit]; + OffsetNumber chainSrc[MaxIndexTuplesPerPageLimit]; + OffsetNumber chainDest[MaxIndexTuplesPerPageLimit]; + OffsetNumber predecessor[MaxIndexTuplesPerPageLimit + 1]; + bool deletable[MaxIndexTuplesPerPageLimit + 1]; int nDeletable; OffsetNumber i, max = PageGetMaxOffsetNumber(page); @@ -408,7 +408,7 @@ vacuumLeafRoot(spgBulkDeleteState *bds, Relation index, Buffer buffer) { Page page = BufferGetPage(buffer); spgxlogVacuumRoot xlrec; - OffsetNumber toDelete[MaxIndexTuplesPerPage]; + OffsetNumber toDelete[MaxIndexTuplesPerPageLimit]; OffsetNumber i, max = PageGetMaxOffsetNumber(page); @@ -498,8 +498,8 @@ vacuumRedirectAndPlaceholder(Relation index, Relation heaprel, Buffer buffer) firstPlaceholder = InvalidOffsetNumber; bool hasNonPlaceholder = false; bool hasUpdate = false; - OffsetNumber itemToPlaceholder[MaxIndexTuplesPerPage]; - OffsetNumber itemnos[MaxIndexTuplesPerPage]; + OffsetNumber itemToPlaceholder[MaxIndexTuplesPerPageLimit]; + OffsetNumber itemnos[MaxIndexTuplesPerPageLimit]; spgxlogVacuumRedirect xlrec; GlobalVisState *vistest; diff --git a/src/backend/access/transam/generic_xlog.c b/src/backend/access/transam/generic_xlog.c index 771dde3419..ba98cad70b 100644 --- a/src/backend/access/transam/generic_xlog.c +++ b/src/backend/access/transam/generic_xlog.c @@ -45,7 +45,7 @@ */ #define FRAGMENT_HEADER_SIZE (2 * sizeof(OffsetNumber)) #define MATCH_THRESHOLD FRAGMENT_HEADER_SIZE -#define MAX_DELTA_SIZE (cluster_block_size + 2 * FRAGMENT_HEADER_SIZE) +#define MAX_DELTA_SIZE (MAX_BLOCK_SIZE + 2 * FRAGMENT_HEADER_SIZE) /* Struct of generic xlog data for single page */ typedef struct diff --git a/src/backend/access/transam/timeline.c b/src/backend/access/transam/timeline.c index 8b4ed3d053..24319d1aeb 100644 --- a/src/backend/access/transam/timeline.c +++ b/src/backend/access/transam/timeline.c @@ -307,7 +307,7 @@ writeTimeLineHistory(TimeLineID newTLI, TimeLineID parentTLI, char path[MAXPGPATH]; char tmppath[MAXPGPATH]; char histfname[MAXFNAMELEN]; - char buffer[cluster_block_size]; + char buffer[DEFAULT_BLOCK_SIZE]; int srcfd; int fd; int nbytes; diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index c7f38f7bc6..032ea78f3d 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -67,6 +67,7 @@ #include "catalog/catversion.h" #include "catalog/pg_control.h" #include "catalog/pg_database.h" +#include "common/blocksize.h" #include "common/controldata_utils.h" #include "common/file_utils.h" #include "executor/instrument.h" @@ -3924,7 +3925,7 @@ WriteControlFile(void) ControlFile->floatFormat = FLOATFORMAT_VALUE; ControlFile->blcksz = cluster_block_size; - ControlFile->relseg_size = RELSEG_SIZE; + ControlFile->relseg_size = cluster_relseg_size; ControlFile->xlog_blcksz = XLOG_BLCKSZ; ControlFile->xlog_seg_size = wal_segment_size; @@ -3990,14 +3991,104 @@ WriteControlFile(void) XLOG_CONTROL_FILE))); } +extern int block_size; + +/* + * This routine reads and returns the block size from the control file as + * needed; since this is the only field we care about and it needs to be + * handled early in the process, we don't worry about locks and the like; this + * is static for the life of the cluster. + */ +uint32 +ClusterBlockSize(void) +{ + pg_crc32c crc; + int fd, r; + ControlFileData cluster_control; + + if (block_size) + return block_size; + + if (ControlFile) + return ControlFile->blcksz; + + // TODO: check for bootstrap mode, since that's passed in the param? + + /* neither shortcut worked, so go ahead and open the control file and + * parse out only basic field */ + fd = BasicOpenFile(XLOG_CONTROL_FILE, + O_RDONLY | PG_BINARY); + if (fd < 0) + ereport(PANIC, + (errcode_for_file_access(), + errmsg("could not open file \"%s\": %m", + XLOG_CONTROL_FILE))); + + r = read(fd, &cluster_control, sizeof(ControlFileData)); + if (r != sizeof(ControlFileData)) + { + if (r < 0) + ereport(PANIC, + (errcode_for_file_access(), + errmsg("could not read file \"%s\": %m", + XLOG_CONTROL_FILE))); + else + ereport(PANIC, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("could not read file \"%s\": read %d of %zu", + XLOG_CONTROL_FILE, r, sizeof(ControlFileData)))); + } + close(fd); + + /* + * Check for expected pg_control format version. If this is wrong, the + * CRC check will likely fail because we'll be checking the wrong number + * of bytes. Complaining about wrong version will probably be more + * enlightening than complaining about wrong CRC. + */ + + if (cluster_control.pg_control_version != PG_CONTROL_VERSION && cluster_control.pg_control_version % 65536 == 0 && cluster_control.pg_control_version / 65536 != 0) + ereport(FATAL, + (errmsg("database files are incompatible with server"), + errdetail("The database cluster was initialized with PG_CONTROL_VERSION %d (0x%08x)," + " but the server was compiled with PG_CONTROL_VERSION %d (0x%08x).", + cluster_control.pg_control_version, cluster_control.pg_control_version, + PG_CONTROL_VERSION, PG_CONTROL_VERSION), + errhint("This could be a problem of mismatched byte ordering. It looks like you need to initdb."))); + + if (cluster_control.pg_control_version != PG_CONTROL_VERSION) + ereport(FATAL, + (errmsg("database files are incompatible with server"), + errdetail("The database cluster was initialized with PG_CONTROL_VERSION %d," + " but the server was compiled with PG_CONTROL_VERSION %d.", + cluster_control.pg_control_version, PG_CONTROL_VERSION), + errhint("It looks like you need to initdb."))); + + /* Now check the CRC. */ + INIT_CRC32C(crc); + COMP_CRC32C(crc, + (char *) &cluster_control, + offsetof(ControlFileData, crc)); + FIN_CRC32C(crc); + + if (!EQ_CRC32C(crc, cluster_control.crc)) + ereport(FATAL, + (errmsg("incorrect checksum in control file"))); + + block_size = cluster_control.blcksz; + + return block_size; +} + static void ReadControlFile(void) { pg_crc32c crc; int fd; static char wal_segsz_str[20]; + static char block_size_str[20]; int r; - + int block_size; /* * Read data... */ @@ -4063,6 +4154,26 @@ ReadControlFile(void) ereport(FATAL, (errmsg("incorrect checksum in control file"))); + /* + * Block size computations affect a number of things that are later + * checked, so ensure that we calculate as soon as CRC has been validated + * before checking other things that may depend on it. + */ + + block_size = ControlFile->blcksz; + + if (!IsValidBlockSize(block_size)) + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg_plural("Block size must be a power of two between 1k and 32k, but the control file specifies %d byte", + "Block size must be a power of two between 1k and 32k, but the control file specifies %d bytes", + block_size, + block_size))); + + BlockSizeInit(block_size); + snprintf(block_size_str, sizeof(block_size_str), "%d", block_size); + SetConfigOption("block_size", block_size_str, PGC_INTERNAL, + PGC_S_DYNAMIC_DEFAULT); + /* * Do compatibility checking immediately. If the database isn't * compatible with the backend executable, we want to abort before we can @@ -4087,19 +4198,12 @@ ReadControlFile(void) (errmsg("database files are incompatible with server"), errdetail("The database cluster appears to use a different floating-point number format than the server executable."), errhint("It looks like you need to initdb."))); - if (ControlFile->blcksz != cluster_block_size) - ereport(FATAL, - (errmsg("database files are incompatible with server"), - errdetail("The database cluster was initialized with cluster_block_size %d," - " but the server was compiled with cluster_block_size %d.", - ControlFile->blcksz, cluster_block_size), - errhint("It looks like you need to recompile or initdb."))); - if (ControlFile->relseg_size != RELSEG_SIZE) + if (ControlFile->relseg_size != cluster_relseg_size) ereport(FATAL, (errmsg("database files are incompatible with server"), - errdetail("The database cluster was initialized with RELSEG_SIZE %d," - " but the server was compiled with RELSEG_SIZE %d.", - ControlFile->relseg_size, RELSEG_SIZE), + errdetail("The database cluster was initialized with cluster_relseg_size %d," + " but the server was compiled with cluster_relseg_size %d.", + ControlFile->relseg_size, cluster_relseg_size), errhint("It looks like you need to recompile or initdb."))); if (ControlFile->xlog_blcksz != XLOG_BLCKSZ) ereport(FATAL, diff --git a/src/backend/access/transam/xloginsert.c b/src/backend/access/transam/xloginsert.c index 87e04d6e03..d833a5734a 100644 --- a/src/backend/access/transam/xloginsert.c +++ b/src/backend/access/transam/xloginsert.c @@ -46,18 +46,18 @@ * backup block image. */ #ifdef USE_LZ4 -#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(cluster_block_size) +#define LZ4_MAX_BLCKSZ LZ4_COMPRESSBOUND(MAX_BLOCK_SIZE) #else #define LZ4_MAX_BLCKSZ 0 #endif #ifdef USE_ZSTD -#define ZSTD_MAX_BLCKSZ ZSTD_COMPRESSBOUND(cluster_block_size) +#define ZSTD_MAX_BLCKSZ ZSTD_COMPRESSBOUND(MAX_BLOCK_SIZE) #else #define ZSTD_MAX_BLCKSZ 0 #endif -#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(cluster_block_size) +#define PGLZ_MAX_BLCKSZ PGLZ_MAX_OUTPUT(MAX_BLOCK_SIZE) /* Buffer size required to store a compressed version of backup block image */ #define COMPRESS_BUFSIZE Max(Max(PGLZ_MAX_BLCKSZ, LZ4_MAX_BLCKSZ), ZSTD_MAX_BLCKSZ) diff --git a/src/backend/backup/basebackup.c b/src/backend/backup/basebackup.c index 8fa1ae018a..a275b186f6 100644 --- a/src/backend/backup/basebackup.c +++ b/src/backend/backup/basebackup.c @@ -23,6 +23,7 @@ #include "backup/basebackup_sink.h" #include "backup/basebackup_target.h" #include "commands/defrem.h" +#include "common/blocksize.h" #include "common/compression.h" #include "common/file_perm.h" #include "lib/stringinfo.h" @@ -370,7 +371,7 @@ perform_base_backup(basebackup_options *opt, bbsink *sink) else { /* Properly terminate the tarfile. */ - StaticAssertDecl(2 * TAR_BLOCK_SIZE <= cluster_block_size, + StaticAssertDecl(2 * TAR_BLOCK_SIZE <= MAX_BLOCK_SIZE, "cluster_block_size too small for 2 tar blocks"); memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE); bbsink_archive_contents(sink, 2 * TAR_BLOCK_SIZE); @@ -623,7 +624,7 @@ perform_base_backup(basebackup_options *opt, bbsink *sink) } /* Properly terminate the tar file. */ - StaticAssertStmt(2 * TAR_BLOCK_SIZE <= cluster_block_size, + StaticAssertStmt(2 * TAR_BLOCK_SIZE <= MAX_BLOCK_SIZE, "cluster_block_size too small for 2 tar blocks"); memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE); bbsink_archive_contents(sink, 2 * TAR_BLOCK_SIZE); @@ -1580,7 +1581,7 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, if (verify_checksum) { - for (i = 0; i < cnt / cluster_block_size; i++) + for (i = 0; i < cnt >> cluster_block_bits; i++) { page = sink->bbs_buffer + cluster_block_size * i; @@ -1594,7 +1595,7 @@ sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, */ if (!PageIsNew(page) && PageGetLSN(page) < sink->bbs_state->startptr) { - checksum = pg_checksum_page((char *) page, blkno + segmentno * RELSEG_SIZE); + checksum = pg_checksum_page((char *) page, blkno + segmentno * RELSEG_SIZE, cluster_block_size); phdr = (PageHeader) page; if (phdr->pd_checksum != checksum) { @@ -1749,8 +1750,8 @@ _tarWriteHeader(bbsink *sink, const char *filename, const char *linktarget, * large enough to fit an entire tar block. We double-check by means * of these assertions. */ - StaticAssertDecl(TAR_BLOCK_SIZE <= cluster_block_size, - "cluster_block_size too small for tar block"); + StaticAssertDecl(TAR_BLOCK_SIZE <= DEFAULT_BLOCK_SIZE, + "DEFAULT_BLOCK_SIZE too small for tar block"); Assert(sink->bbs_buffer_length >= TAR_BLOCK_SIZE); rc = tarCreateHeader(sink->bbs_buffer, filename, linktarget, diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index 5810f8825e..91ace7613d 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -28,6 +28,7 @@ #include "catalog/index.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" +#include "common/blocksize.h" #include "common/link-canary.h" #include "libpq/pqsignal.h" #include "miscadmin.h" @@ -46,6 +47,7 @@ #include "utils/relmapper.h" uint32 bootstrap_data_checksum_version = 0; /* No checksum */ +uint32 bootstrap_blocksize = DEFAULT_BLOCK_SIZE; static void CheckerModeMain(void); @@ -221,10 +223,18 @@ BootstrapModeMain(int argc, char *argv[], bool check_only) argv++; argc--; - while ((flag = getopt(argc, argv, "B:c:d:D:Fkr:X:-:")) != -1) + while ((flag = getopt(argc, argv, "b:B:c:d:D:Fkr:X:-:")) != -1) { switch (flag) { + case 'b': + bootstrap_blocksize = strtol(optarg, NULL, 0); + if (!IsValidBlockSize(bootstrap_blocksize)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid block size: %s; must be power of two between 1k and 32k", + optarg))); + break; case 'B': SetConfigOption("shared_buffers", optarg, PGC_POSTMASTER, PGC_S_ARGV); break; @@ -300,6 +310,8 @@ BootstrapModeMain(int argc, char *argv[], bool check_only) if (!SelectConfigFiles(userDoption, progname)) proc_exit(1); + BlockSizeInit(bootstrap_blocksize); + /* * Validate we have been given a reasonable-looking DataDir and change * into it diff --git a/src/backend/commands/async.c b/src/backend/commands/async.c index c7d19f4ee0..6a1ae24955 100644 --- a/src/backend/commands/async.c +++ b/src/backend/commands/async.c @@ -163,7 +163,7 @@ * than that, so changes in that data structure won't affect user-visible * restrictions. */ -#define NOTIFY_PAYLOAD_MAX_LENGTH (cluster_block_size - NAMEDATALEN - 128) +#define NOTIFY_PAYLOAD_MAX_LENGTH (DEFAULT_BLOCK_SIZE - NAMEDATALEN - 128) /* * Struct representing an entry in the global notify queue @@ -311,7 +311,7 @@ static AsyncQueueControl *asyncQueueControl; static SlruCtlData NotifyCtlData; #define NotifyCtl (&NotifyCtlData) -#define QUEUE_PAGESIZE cluster_block_size +#define QUEUE_PAGESIZE DEFAULT_BLOCK_SIZE #define QUEUE_FULL_WARN_INTERVAL 5000 /* warn at most once every 5s */ /* diff --git a/src/backend/nodes/tidbitmap.c b/src/backend/nodes/tidbitmap.c index 971c29c428..5c7593617a 100644 --- a/src/backend/nodes/tidbitmap.c +++ b/src/backend/nodes/tidbitmap.c @@ -53,7 +53,7 @@ * the per-page bitmaps variable size. We just legislate that the size * is this: */ -#define MAX_TUPLES_PER_PAGE MaxHeapTuplesPerPage +#define MAX_TUPLES_PER_PAGE MaxHeapTuplesPerPageLimit /* * When we have to switch over to lossy storage, we use a data structure @@ -70,7 +70,7 @@ * too different. But we also want PAGES_PER_CHUNK to be a power of 2 to * avoid expensive integer remainder operations. So, define it like this: */ -#define PAGES_PER_CHUNK (cluster_block_size / 32) +#define PAGES_PER_CHUNK (MAX_BLOCK_SIZE / 32) /* We use BITS_PER_BITMAPWORD and typedef bitmapword from nodes/bitmapset.h */ diff --git a/src/backend/po/de.po b/src/backend/po/de.po index ac2cb6e4cf..be75aea0a4 100644 --- a/src/backend/po/de.po +++ b/src/backend/po/de.po @@ -22030,7 +22030,7 @@ msgstr "debug_io_direct wird für WAL nicht unterstützt, weil XLOG_BLCKSZ zu kl #: storage/file/fd.c:3951 #, c-format msgid "debug_io_direct is not supported for data because cluster_block_size is too small" -msgstr "debug_io_direct wird für Daten nicht unterstützt, weil cluster_block_size zu klein ist" +msgstr "debug_io_direct wird für Daten nicht unterstützt, weil cluster_block_sizes zu klein ist" #: storage/file/reinit.c:145 #, c-format diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index d11f73c13b..cdac86cd20 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -3500,7 +3500,7 @@ RelationGetNumberOfBlocksInFork(Relation relation, ForkNumber forkNum) szbytes = table_relation_size(relation, forkNum); - return (szbytes + (cluster_block_size - 1)) / cluster_block_size; + return (szbytes + (cluster_block_size - 1)) >> cluster_block_bits; } else if (RELKIND_HAS_STORAGE(relation->rd_rel->relkind)) { diff --git a/src/backend/storage/buffer/localbuf.c b/src/backend/storage/buffer/localbuf.c index 935b870933..efb9cccc44 100644 --- a/src/backend/storage/buffer/localbuf.c +++ b/src/backend/storage/buffer/localbuf.c @@ -745,7 +745,7 @@ GetLocalBufferStorage(void) /* But not more than what we need for all remaining local bufs */ num_bufs = Min(num_bufs, NLocBuffer - total_bufs_allocated); /* And don't overflow MaxAllocSize, either */ - num_bufs = Min(num_bufs, MaxAllocSize / cluster_block_size); + num_bufs = Min(num_bufs, MaxAllocSize >> cluster_block_bits); /* Buffers should be I/O aligned. */ cur_block = (char *) diff --git a/src/backend/storage/file/buffile.c b/src/backend/storage/file/buffile.c index cf5d980026..d817a6ec5a 100644 --- a/src/backend/storage/file/buffile.c +++ b/src/backend/storage/file/buffile.c @@ -60,7 +60,7 @@ * tablespaces when available. */ #define MAX_PHYSICAL_FILESIZE 0x40000000 -#define BUFFILE_SEG_SIZE (MAX_PHYSICAL_FILESIZE / cluster_block_size) +#define BUFFILE_SEG_SIZE (MAX_PHYSICAL_FILESIZE >> cluster_block_bits) /* * This data structure represents a buffered file that consists of one or @@ -867,7 +867,7 @@ BufFileTellBlock(BufFile *file) { long blknum; - blknum = (file->curOffset + file->pos) / cluster_block_size; + blknum = (file->curOffset + file->pos) >> cluster_block_bits; blknum += file->curFile * BUFFILE_SEG_SIZE; return blknum; } diff --git a/src/backend/storage/file/fd.c b/src/backend/storage/file/fd.c index 7207405e06..c04166d19d 100644 --- a/src/backend/storage/file/fd.c +++ b/src/backend/storage/file/fd.c @@ -3943,10 +3943,10 @@ check_debug_io_direct(char **newval, void **extra, GucSource source) result = false; } #endif -#if cluster_block_size < PG_IO_ALIGN_SIZE +#if DEFAULT_BLOCK_SIZE < PG_IO_ALIGN_SIZE if (result && (flags & IO_DIRECT_DATA)) { - GUC_check_errdetail("debug_io_direct is not supported for data because cluster_block_size is too small"); + GUC_check_errdetail("debug_io_direct is not supported for data because DEFAULT_BLOCK_SIZE is too small"); result = false; } #endif diff --git a/src/backend/storage/freespace/freespace.c b/src/backend/storage/freespace/freespace.c index f17fe758d6..c89ed0c23f 100644 --- a/src/backend/storage/freespace/freespace.c +++ b/src/backend/storage/freespace/freespace.c @@ -87,8 +87,8 @@ typedef struct int logpageno; /* page number within the level */ } FSMAddress; -/* Address of the root page. */ -static const FSMAddress FSM_ROOT_ADDRESS = {FSM_ROOT_LEVEL, 0}; +/* Address of the root page. Level is adjusted by FreeSpaceMapInit() */ +static FSMAddress FSM_ROOT_ADDRESS = {0,0}; /* functions to navigate the tree */ static FSMAddress fsm_get_child(FSMAddress parent, uint16 slot); @@ -116,6 +116,17 @@ static uint8 fsm_vacuum_page(Relation rel, FSMAddress addr, /******** Public API ********/ +/* + * FreeSpaceMapInit - initialize the FSM system with the cluster block size + * + */ + +void +FreeSpaceMapInit(void) +{ + FSM_ROOT_ADDRESS.level = FSM_ROOT_LEVEL; +} + /* * GetPageWithFreeSpace - try to find a page in the given relation with * at least the specified amount of free space. diff --git a/src/backend/storage/large_object/inv_api.c b/src/backend/storage/large_object/inv_api.c index 17b834d39b..b178213391 100644 --- a/src/backend/storage/large_object/inv_api.c +++ b/src/backend/storage/large_object/inv_api.c @@ -598,7 +598,7 @@ inv_write(LargeObjectDesc *obj_desc, const char *buf, int nbytes) { bytea hdr; /* this is to make the union big enough for a LO data chunk: */ - char data[cluster_loblksize + VARHDRSZ]; + char data[LOBLKSIZE_LIMIT + VARHDRSZ]; /* ensure union is aligned well enough: */ int32 align_it; } workbuf; @@ -789,7 +789,7 @@ inv_truncate(LargeObjectDesc *obj_desc, int64 len) { bytea hdr; /* this is to make the union big enough for a LO data chunk: */ - char data[cluster_loblksize + VARHDRSZ]; + char data[LOBLKSIZE_LIMIT + VARHDRSZ]; /* ensure union is aligned well enough: */ int32 align_it; } workbuf; diff --git a/src/backend/storage/page/bufpage.c b/src/backend/storage/page/bufpage.c index 07a664355f..6c6217bb00 100644 --- a/src/backend/storage/page/bufpage.c +++ b/src/backend/storage/page/bufpage.c @@ -102,7 +102,7 @@ PageIsVerifiedExtended(Page page, BlockNumber blkno, int flags) { if (DataChecksumsEnabled()) { - checksum = pg_checksum_page((char *) page, blkno); + checksum = pg_checksum_page((char *) page, blkno, cluster_block_size); if (checksum != p->pd_checksum) checksum_failure = true; @@ -702,7 +702,7 @@ PageRepairFragmentation(Page page) Offset pd_upper = ((PageHeader) page)->pd_upper; Offset pd_special = ((PageHeader) page)->pd_special; Offset last_offset; - itemIdCompactData itemidbase[MaxHeapTuplesPerPage]; + itemIdCompactData itemidbase[MaxHeapTuplesPerPageLimit]; itemIdCompact itemidptr; ItemId lp; int nline, @@ -1165,8 +1165,8 @@ PageIndexMultiDelete(Page page, OffsetNumber *itemnos, int nitems) Offset pd_upper = phdr->pd_upper; Offset pd_special = phdr->pd_special; Offset last_offset; - itemIdCompactData itemidbase[MaxIndexTuplesPerPage]; - ItemIdData newitemids[MaxIndexTuplesPerPage]; + itemIdCompactData itemidbase[MaxIndexTuplesPerPageLimit]; + ItemIdData newitemids[MaxIndexTuplesPerPageLimit]; itemIdCompact itemidptr; ItemId lp; int nline, @@ -1528,7 +1528,7 @@ PageSetChecksumCopy(Page page, BlockNumber blkno) 0); memcpy(pageCopy, (char *) page, cluster_block_size); - ((PageHeader) pageCopy)->pd_checksum = pg_checksum_page(pageCopy, blkno); + ((PageHeader) pageCopy)->pd_checksum = pg_checksum_page(pageCopy, blkno, cluster_block_size); return pageCopy; } @@ -1545,5 +1545,5 @@ PageSetChecksumInplace(Page page, BlockNumber blkno) if (PageIsNew(page) || !DataChecksumsEnabled()) return; - ((PageHeader) page)->pd_checksum = pg_checksum_page((char *) page, blkno); + ((PageHeader) page)->pd_checksum = pg_checksum_page((char *) page, blkno, cluster_block_size); } diff --git a/src/backend/storage/smgr/README b/src/backend/storage/smgr/README index cf3aa5632b..22515ab823 100644 --- a/src/backend/storage/smgr/README +++ b/src/backend/storage/smgr/README @@ -50,3 +50,55 @@ to relfilelocator and block number, to identify which relation fork you want to access. Since most code wants to access the main fork, a shortcut version of ReadBuffer that accesses MAIN_FORKNUM is provided in the buffer manager for convenience. + + +Adjustments for Variable Block Sizes +==================================== + +Prior to 17, the disk block size (aka cluster_block_size) was set at compile-time. While +each database cluster still has a fixed block size, this is now selected at +`initdb` time, meaning that a single PostgreSQL binary installation can support +clusters of varying sizes. + +As this is now a per-cluster setting, the block size is set in the pg_control +file. There are multiple computed settings which rely on the cluster block size, +so these also need to be initialized at cluster start. This is managed via +BlockSizeInit() in src/common/blocksize.c. This routine computes a table for +all values which would have been computed at compile time. + +The expressions used to calculate the values in this table have been moved from +their original locations and parameterized based on blocksize; these are named +CalcXXXX(blocksize), and are stored in src/include/common/blocksize.h. + +In order to minimize code changes, we have kept the names of former constants +(such as MaxHeapTuplesPerPage) and replaced their definitions with an +appropriate lookup in this table. Since these lookups are now handled at +runtime, it is important to initialize these as early as possible so any code +usage of these constants uses the computed value after it has been initialized +instead of 0. This applies to any piece of the code which needs to know about +the block size or uses any value which was based on it. + +To ensure that any code that depended on the old behavior is properly adjusted, +we have removed cluster_block_size as a constant, which will ensure that external code will +fail to compile. When writing new code which cases about the block size, you +should use one of the following values: + +- cluster_block_size - evaluates to the cluster's block size as initialized by + BlockSizeInit(). + +- DEFAULT_BLOCK_SIZE - the compiled-in default block size; 8k + +- MIN_BLOCK_SIZE - the smallest supported block size; 1k + +- MAX_BLOCK_SIZE - the largest supported block size; 32k + +In general, you should use cluster_block_size unless you are allocating space on +the stack, and even then you should be careful to allocate based on +MAX_BLOCK_SIZE, but still limit accesses to these structs based on the /actual/ +runtime cluster block size. The core code has bene modified to follow these +standards. + +Occasionally, we had been using cluster_block_size as a "big enough buffer" rather than +caring about any specific size correlated with the cluster. In those cases, you +should just use DEFAULT_BLOCK_SIZE. + diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c index 319e8d8655..15f1d9849e 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -47,13 +47,13 @@ * easier to support relations that are larger than the operating * system's file size limit (often 2GBytes). In order to do that, * we break relations up into "segment" files that are each shorter than - * the OS file size limit. The segment size is set by the RELSEG_SIZE + * the OS file size limit. The segment size is set by the cluster_relseg_size * configuration constant in pg_config.h. * * On disk, a relation must consist of consecutively numbered segment * files in the pattern - * -- Zero or more full segments of exactly RELSEG_SIZE blocks each - * -- Exactly one partial segment of size 0 <= size < RELSEG_SIZE blocks + * -- Zero or more full segments of exactly cluster_relseg_size blocks each + * -- Exactly one partial segment of size 0 <= size < cluster_relseg_size blocks * -- Optionally, any number of inactive segments of size 0 blocks. * The full and partial segments are collectively the "active" segments. * Inactive segments are those that once contained data but are currently @@ -110,7 +110,7 @@ static MemoryContext MdCxt; /* context for all MdfdVec objects */ #define EXTENSION_CREATE_RECOVERY (1 << 3) /* * Allow opening segments which are preceded by segments smaller than - * RELSEG_SIZE, e.g. inactive segments (see above). Note that this breaks + * cluster_relseg_size, e.g. inactive segments (see above). Note that this breaks * mdnblocks() and related functionality henceforth - which currently is ok, * because this is only required in the checkpointer which never uses * mdnblocks(). @@ -490,9 +490,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, v = _mdfd_getseg(reln, forknum, blocknum, skipFsync, EXTENSION_CREATE); - seekpos = (off_t) cluster_block_size * (blocknum % ((BlockNumber) RELSEG_SIZE)); + seekpos = (off_t) cluster_block_size * (blocknum % ((BlockNumber) cluster_relseg_size)); - Assert(seekpos < (off_t) cluster_block_size * RELSEG_SIZE); + Assert(seekpos < (off_t) cluster_block_size * cluster_relseg_size); if ((nbytes = FileWrite(v->mdfd_vfd, buffer, cluster_block_size, seekpos, WAIT_EVENT_DATA_FILE_EXTEND)) != cluster_block_size) { @@ -514,7 +514,7 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); - Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) cluster_relseg_size)); } /* @@ -552,19 +552,19 @@ mdzeroextend(SMgrRelation reln, ForkNumber forknum, while (remblocks > 0) { - BlockNumber segstartblock = curblocknum % ((BlockNumber) RELSEG_SIZE); + BlockNumber segstartblock = curblocknum % ((BlockNumber) cluster_relseg_size); off_t seekpos = (off_t) cluster_block_size * segstartblock; int numblocks; - if (segstartblock + remblocks > RELSEG_SIZE) - numblocks = RELSEG_SIZE - segstartblock; + if (segstartblock + remblocks > cluster_relseg_size) + numblocks = cluster_relseg_size - segstartblock; else numblocks = remblocks; v = _mdfd_getseg(reln, forknum, curblocknum, skipFsync, EXTENSION_CREATE); - Assert(segstartblock < RELSEG_SIZE); - Assert(segstartblock + numblocks <= RELSEG_SIZE); + Assert(segstartblock < cluster_relseg_size); + Assert(segstartblock + numblocks <= cluster_relseg_size); /* * If available and useful, use posix_fallocate() (via @@ -618,7 +618,7 @@ mdzeroextend(SMgrRelation reln, ForkNumber forknum, if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); - Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) cluster_relseg_size)); remblocks -= numblocks; curblocknum += numblocks; @@ -670,7 +670,7 @@ mdopenfork(SMgrRelation reln, ForkNumber forknum, int behavior) mdfd->mdfd_vfd = fd; mdfd->mdfd_segno = 0; - Assert(_mdnblocks(reln, forknum, mdfd) <= ((BlockNumber) RELSEG_SIZE)); + Assert(_mdnblocks(reln, forknum, mdfd) <= ((BlockNumber) cluster_relseg_size)); return mdfd; } @@ -726,9 +726,9 @@ mdprefetch(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum) if (v == NULL) return false; - seekpos = (off_t) cluster_block_size * (blocknum % ((BlockNumber) RELSEG_SIZE)); + seekpos = (off_t) cluster_block_size * (blocknum % ((BlockNumber) cluster_relseg_size)); - Assert(seekpos < (off_t) cluster_block_size * RELSEG_SIZE); + Assert(seekpos < (off_t) cluster_block_size * cluster_relseg_size); (void) FilePrefetch(v->mdfd_vfd, seekpos, cluster_block_size, WAIT_EVENT_DATA_FILE_PREFETCH); #endif /* USE_PREFETCH */ @@ -760,9 +760,9 @@ mdread(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, v = _mdfd_getseg(reln, forknum, blocknum, false, EXTENSION_FAIL | EXTENSION_CREATE_RECOVERY); - seekpos = (off_t) cluster_block_size * (blocknum % ((BlockNumber) RELSEG_SIZE)); + seekpos = (off_t) cluster_block_size * (blocknum % ((BlockNumber) cluster_relseg_size)); - Assert(seekpos < (off_t) cluster_block_size * RELSEG_SIZE); + Assert(seekpos < (off_t) cluster_block_size * cluster_relseg_size); nbytes = FileRead(v->mdfd_vfd, buffer, cluster_block_size, seekpos, WAIT_EVENT_DATA_FILE_READ); @@ -834,9 +834,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, v = _mdfd_getseg(reln, forknum, blocknum, skipFsync, EXTENSION_FAIL | EXTENSION_CREATE_RECOVERY); - seekpos = (off_t) cluster_block_size * (blocknum % ((BlockNumber) RELSEG_SIZE)); + seekpos = (off_t) cluster_block_size * (blocknum % ((BlockNumber) cluster_relseg_size)); - Assert(seekpos < (off_t) cluster_block_size * RELSEG_SIZE); + Assert(seekpos < (off_t) cluster_block_size * cluster_relseg_size); nbytes = FileWrite(v->mdfd_vfd, buffer, cluster_block_size, seekpos, WAIT_EVENT_DATA_FILE_WRITE); @@ -907,17 +907,17 @@ mdwriteback(SMgrRelation reln, ForkNumber forknum, return; /* compute offset inside the current segment */ - segnum_start = blocknum / RELSEG_SIZE; + segnum_start = blocknum / cluster_relseg_size; /* compute number of desired writes within the current segment */ - segnum_end = (blocknum + nblocks - 1) / RELSEG_SIZE; + segnum_end = (blocknum + nblocks - 1) / cluster_relseg_size; if (segnum_start != segnum_end) - nflush = RELSEG_SIZE - (blocknum % ((BlockNumber) RELSEG_SIZE)); + nflush = cluster_relseg_size - (blocknum % ((BlockNumber) cluster_relseg_size)); Assert(nflush >= 1); Assert(nflush <= nblocks); - seekpos = (off_t) cluster_block_size * (blocknum % ((BlockNumber) RELSEG_SIZE)); + seekpos = (off_t) cluster_block_size * (blocknum % ((BlockNumber) cluster_relseg_size)); FileWriteback(v->mdfd_vfd, seekpos, (off_t) cluster_block_size * nflush, WAIT_EVENT_DATA_FILE_FLUSH); @@ -948,7 +948,7 @@ mdnblocks(SMgrRelation reln, ForkNumber forknum) /* * Start from the last open segments, to avoid redundant seeks. We have - * previously verified that these segments are exactly RELSEG_SIZE long, + * previously verified that these segments are exactly cluster_relseg_size long, * and it's useless to recheck that each time. * * NOTE: this assumption could only be wrong if another backend has @@ -965,13 +965,13 @@ mdnblocks(SMgrRelation reln, ForkNumber forknum) for (;;) { nblocks = _mdnblocks(reln, forknum, v); - if (nblocks > ((BlockNumber) RELSEG_SIZE)) + if (nblocks > ((BlockNumber) cluster_relseg_size)) elog(FATAL, "segment too big"); - if (nblocks < ((BlockNumber) RELSEG_SIZE)) - return (segno * ((BlockNumber) RELSEG_SIZE)) + nblocks; + if (nblocks < ((BlockNumber) cluster_relseg_size)) + return (segno * ((BlockNumber) cluster_relseg_size)) + nblocks; /* - * If segment is exactly RELSEG_SIZE, advance to next one. + * If segment is exactly cluster_relseg_size, advance to next one. */ segno++; @@ -984,7 +984,7 @@ mdnblocks(SMgrRelation reln, ForkNumber forknum) */ v = _mdfd_openseg(reln, forknum, segno, 0); if (v == NULL) - return segno * ((BlockNumber) RELSEG_SIZE); + return segno * ((BlockNumber) cluster_relseg_size); } } @@ -1025,7 +1025,7 @@ mdtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks) { MdfdVec *v; - priorblocks = (curopensegs - 1) * RELSEG_SIZE; + priorblocks = (curopensegs - 1) * cluster_relseg_size; v = &reln->md_seg_fds[forknum][curopensegs - 1]; @@ -1050,12 +1050,12 @@ mdtruncate(SMgrRelation reln, ForkNumber forknum, BlockNumber nblocks) FileClose(v->mdfd_vfd); _fdvec_resize(reln, forknum, curopensegs - 1); } - else if (priorblocks + ((BlockNumber) RELSEG_SIZE) > nblocks) + else if (priorblocks + ((BlockNumber) cluster_relseg_size) > nblocks) { /* * This is the last segment we want to keep. Truncate the file to * the right length. NOTE: if nblocks is exactly a multiple K of - * RELSEG_SIZE, we will truncate the K+1st segment to 0 length but + * cluster_relseg_size, we will truncate the K+1st segment to 0 length but * keep it. This adheres to the invariant given in the header * comments. */ @@ -1372,7 +1372,7 @@ _mdfd_openseg(SMgrRelation reln, ForkNumber forknum, BlockNumber segno, v->mdfd_vfd = fd; v->mdfd_segno = segno; - Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) cluster_relseg_size)); /* all done */ return v; @@ -1399,7 +1399,7 @@ _mdfd_getseg(SMgrRelation reln, ForkNumber forknum, BlockNumber blkno, (EXTENSION_FAIL | EXTENSION_CREATE | EXTENSION_RETURN_NULL | EXTENSION_DONT_OPEN)); - targetseg = blkno / ((BlockNumber) RELSEG_SIZE); + targetseg = blkno / ((BlockNumber) cluster_relseg_size); /* if an existing and opened segment, we're done */ if (targetseg < reln->md_num_open_segs[forknum]) @@ -1436,7 +1436,7 @@ _mdfd_getseg(SMgrRelation reln, ForkNumber forknum, BlockNumber blkno, Assert(nextsegno == v->mdfd_segno + 1); - if (nblocks > ((BlockNumber) RELSEG_SIZE)) + if (nblocks > ((BlockNumber) cluster_relseg_size)) elog(FATAL, "segment too big"); if ((behavior & EXTENSION_CREATE) || @@ -1451,30 +1451,30 @@ _mdfd_getseg(SMgrRelation reln, ForkNumber forknum, BlockNumber blkno, * ahead and create the segments so we can finish out the replay. * * We have to maintain the invariant that segments before the last - * active segment are of size RELSEG_SIZE; therefore, if + * active segment are of size cluster_relseg_size; therefore, if * extending, pad them out with zeroes if needed. (This only * matters if in recovery, or if the caller is extending the * relation discontiguously, but that can happen in hash indexes.) */ - if (nblocks < ((BlockNumber) RELSEG_SIZE)) + if (nblocks < ((BlockNumber) cluster_relseg_size)) { char *zerobuf = palloc_aligned(cluster_block_size, PG_IO_ALIGN_SIZE, MCXT_ALLOC_ZERO); mdextend(reln, forknum, - nextsegno * ((BlockNumber) RELSEG_SIZE) - 1, + nextsegno * ((BlockNumber) cluster_relseg_size) - 1, zerobuf, skipFsync); pfree(zerobuf); } flags = O_CREAT; } else if (!(behavior & EXTENSION_DONT_CHECK_SIZE) && - nblocks < ((BlockNumber) RELSEG_SIZE)) + nblocks < ((BlockNumber) cluster_relseg_size)) { /* * When not extending (or explicitly including truncated * segments), only open the next segment if the current one is - * exactly RELSEG_SIZE. If not (this branch), either return NULL + * exactly cluster_relseg_size. If not (this branch), either return NULL * or fail. */ if (behavior & EXTENSION_RETURN_NULL) @@ -1529,7 +1529,7 @@ _mdnblocks(SMgrRelation reln, ForkNumber forknum, MdfdVec *seg) errmsg("could not seek to end of file \"%s\": %m", FilePathName(seg->mdfd_vfd)))); /* note that this calculation will ignore any partial block at EOF */ - return (BlockNumber) (len / cluster_block_size); + return (BlockNumber) (len >> cluster_block_bits); } /* diff --git a/src/backend/utils/init/miscinit.c b/src/backend/utils/init/miscinit.c index 37d91cb902..da82e96496 100644 --- a/src/backend/utils/init/miscinit.c +++ b/src/backend/utils/init/miscinit.c @@ -1501,8 +1501,8 @@ AddToDataDirLockFile(int target_line, const char *str) int lineno; char *srcptr; char *destptr; - char srcbuffer[cluster_block_size]; - char destbuffer[cluster_block_size]; + char srcbuffer[DEFAULT_BLOCK_SIZE]; + char destbuffer[DEFAULT_BLOCK_SIZE]; fd = open(DIRECTORY_LOCK_FILE, O_RDWR | PG_BINARY, 0); if (fd < 0) @@ -1626,7 +1626,7 @@ RecheckDataDirLockFile(void) int fd; int len; long file_pid; - char buffer[cluster_block_size]; + char buffer[DEFAULT_BLOCK_SIZE]; fd = open(DIRECTORY_LOCK_FILE, O_RDWR | PG_BINARY, 0); if (fd < 0) diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 1ec0edd6ac..98d78c8792 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -30,6 +30,7 @@ #include "access/xact.h" #include "access/xlog.h" +#include "access/xlog_internal.h" #include "catalog/objectaccess.h" #include "catalog/pg_authid.h" #include "catalog/pg_parameter_acl.h" @@ -105,8 +106,8 @@ typedef struct } unit_conversion; /* Ensure that the constants in the tables don't overflow or underflow */ -#if cluster_block_size < 1024 || cluster_block_size > (1024*1024) -#error cluster_block_size must be between 1KB and 1MB +#if DEFAULT_BLOCK_SIZE < 1024 || DEFAULT_BLOCK_SIZE > (1024*1024) +#error DEFAULT_BLOCK_SIZE must be between 1KB and 1MB #endif #if XLOG_BLCKSZ < 1024 || XLOG_BLCKSZ > (1024*1024) #error XLOG_BLCKSZ must be between 1KB and 1MB @@ -134,11 +135,11 @@ static const unit_conversion memory_unit_conversion_table[] = {"kB", GUC_UNIT_MB, 1.0 / 1024.0}, {"B", GUC_UNIT_MB, 1.0 / (1024.0 * 1024.0)}, - {"TB", GUC_UNIT_BLOCKS, (1024.0 * 1024.0 * 1024.0) / (cluster_block_size / 1024)}, - {"GB", GUC_UNIT_BLOCKS, (1024.0 * 1024.0) / (cluster_block_size / 1024)}, - {"MB", GUC_UNIT_BLOCKS, 1024.0 / (cluster_block_size / 1024)}, - {"kB", GUC_UNIT_BLOCKS, 1.0 / (cluster_block_size / 1024)}, - {"B", GUC_UNIT_BLOCKS, 1.0 / cluster_block_size}, + {"TB", GUC_UNIT_BLOCKS, (1024.0 * 1024.0 * 1024.0) / (DEFAULT_BLOCK_SIZE / 1024)}, + {"GB", GUC_UNIT_BLOCKS, (1024.0 * 1024.0) / (DEFAULT_BLOCK_SIZE / 1024)}, + {"MB", GUC_UNIT_BLOCKS, 1024.0 / (DEFAULT_BLOCK_SIZE / 1024)}, + {"kB", GUC_UNIT_BLOCKS, 1.0 / (DEFAULT_BLOCK_SIZE / 1024)}, + {"B", GUC_UNIT_BLOCKS, 1.0 / DEFAULT_BLOCK_SIZE}, {"TB", GUC_UNIT_XBLOCKS, (1024.0 * 1024.0 * 1024.0) / (XLOG_BLCKSZ / 1024)}, {"GB", GUC_UNIT_XBLOCKS, (1024.0 * 1024.0) / (XLOG_BLCKSZ / 1024)}, @@ -1503,6 +1504,16 @@ InitializeGUCOptions(void) */ pg_timezone_initialize(); + /* Load our block size -- no valid in Bootstrap or init mode, since control file hasn't been written */ + if (!(IsInitProcessingMode() || IsBootstrapProcessingMode())) + { + BlockSizeInit(ClusterBlockSize()); + /* + * Ensure GUCs with dynamic limits have been properly initialized + */ + update_dynamic_gucs(); + } + /* * Create GUCMemoryContext and build hash table of all GUC variables. */ diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index a465f7f9d1..fb8c47c406 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -41,6 +41,7 @@ #include "commands/trigger.h" #include "commands/user.h" #include "commands/vacuum.h" +#include "common/blocksize.h" #include "common/scram-common.h" #include "jit/jit.h" #include "libpq/auth.h" @@ -504,8 +505,11 @@ bool log_btree_build_stats = false; char *event_source; bool row_security; + bool check_function_bodies = true; +int block_size; + /* * This GUC exists solely for backward compatibility, check its definition for * details. @@ -588,7 +592,6 @@ static char *session_authorization_string; static int max_function_args; static int max_index_keys; static int max_identifier_length; -static int block_size; static int segment_size; static int shared_memory_size_mb; static int shared_memory_size_in_huge_pages; @@ -3118,7 +3121,7 @@ struct config_int ConfigureNamesInt[] = GUC_NOT_IN_SAMPLE | GUC_DISALLOW_IN_FILE }, &block_size, - cluster_block_size, cluster_block_size, cluster_block_size, + DEFAULT_BLOCK_SIZE, MIN_BLOCK_SIZE, MAX_BLOCK_SIZE, NULL, NULL, NULL }, @@ -3368,7 +3371,8 @@ struct config_int ConfigureNamesInt[] = GUC_UNIT_BLOCKS | GUC_EXPLAIN, }, &min_parallel_table_scan_size, - (8 * 1024 * 1024) / cluster_block_size, 0, INT_MAX / 3, + /* This is set dynamically based on cluster_block_size, but works for a default */ + (8 * 1024 * 1024) / DEFAULT_BLOCK_SIZE, 0, INT_MAX / 3, NULL, NULL, NULL }, @@ -3379,7 +3383,8 @@ struct config_int ConfigureNamesInt[] = GUC_UNIT_BLOCKS | GUC_EXPLAIN, }, &min_parallel_index_scan_size, - (512 * 1024) / cluster_block_size, 0, INT_MAX / 3, + /* This is set dynamically based on cluster_block_size, but works for a default */ + (512 * 1024) / DEFAULT_BLOCK_SIZE, 0, INT_MAX / 3, NULL, NULL, NULL }, @@ -4986,3 +4991,29 @@ struct config_enum ConfigureNamesEnum[] = {NULL, 0, 0, NULL, NULL}, NULL, 0, NULL, NULL, NULL, NULL } }; + + + +/* + * Update the specific GUCs which have dynamic limits. Due to having variable + * block sizes, what were originally constants now depend on the runtime block + * size, so handle the adjustments to those boot values here. + */ +void +update_dynamic_gucs(void) +{ + int i; + + for (i = 0; ConfigureNamesInt[i].gen.name; i++) + { + if (strcmp("min_parallel_table_scan_size", ConfigureNamesInt[i].gen.name) == 0) + { + ConfigureNamesInt[i].boot_val = (8 * 1024 * 1024) >> cluster_block_bits; + } + else if (strcmp("min_parallel_index_scan_size", ConfigureNamesInt[i].gen.name) == 0) + { + ConfigureNamesInt[i].boot_val = (512 * 1024) >> cluster_block_bits; + } + } + +} diff --git a/src/backend/utils/sort/logtape.c b/src/backend/utils/sort/logtape.c index f46d2bac60..9a53205a58 100644 --- a/src/backend/utils/sort/logtape.c +++ b/src/backend/utils/sort/logtape.c @@ -640,7 +640,7 @@ LogicalTapeImport(LogicalTapeSet *lts, int worker, TapeShare *shared) } /* Don't allocate more for read buffer than could possibly help */ lt->max_size = Min(MaxAllocSize, filesize); - tapeblocks = filesize / cluster_block_size; + tapeblocks = filesize >> cluster_block_bits; /* * Update # of allocated blocks and # blocks written to reflect the diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 84b0cc7954..ef810b0f59 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -69,6 +69,7 @@ #include "catalog/pg_class_d.h" /* pgrminclude ignore */ #include "catalog/pg_collation_d.h" #include "catalog/pg_database_d.h" /* pgrminclude ignore */ +#include "common/blocksize.h" #include "common/file_perm.h" #include "common/file_utils.h" #include "common/logging.h" @@ -165,6 +166,8 @@ static bool show_setting = false; static bool data_checksums = false; static char *xlog_dir = NULL; static int wal_segment_size_mb = (DEFAULT_XLOG_SEG_SIZE) / (1024 * 1024); +static char *str_block_size = NULL; +int block_size = DEFAULT_BLOCK_SIZE; /* internal vars */ @@ -1108,8 +1111,8 @@ test_config_settings(void) for (i = 0; i < bufslen; i++) { - /* Use same amount of memory, independent of cluster_block_size */ - test_buffs = (trial_bufs[i] * 8192) / cluster_block_size; + /* Use same amount of memory, independent of block_size */ + test_buffs = (trial_bufs[i] * 8192) / block_size; if (test_buffs <= ok_buffers) { test_buffs = ok_buffers; @@ -1121,10 +1124,10 @@ test_config_settings(void) } n_buffers = test_buffs; - if ((n_buffers * (cluster_block_size / 1024)) % 1024 == 0) - printf("%dMB\n", (n_buffers * (cluster_block_size / 1024)) / 1024); + if ((n_buffers * (block_size / 1024)) % 1024 == 0) + printf("%dMB\n", (n_buffers * (block_size / 1024)) / 1024); else - printf("%dkB\n", n_buffers * (cluster_block_size / 1024)); + printf("%dkB\n", n_buffers * (block_size / 1024)); printf(_("selecting default time zone ... ")); fflush(stdout); @@ -1147,11 +1150,12 @@ test_specific_config_settings(int test_conns, int test_buffs) /* Set up the test postmaster invocation */ printfPQExpBuffer(&cmd, - "\"%s\" --check %s %s " + "\"%s\" --check %s %s -b %d " "-c max_connections=%d " "-c shared_buffers=%d " "-c dynamic_shared_memory_type=%s", backend_exec, boot_options, extra_options, + block_size, test_conns, test_buffs, dynamic_shared_memory_type); @@ -1216,12 +1220,12 @@ setup_config(void) conflines = replace_guc_value(conflines, "max_connections", repltok, false); - if ((n_buffers * (cluster_block_size / 1024)) % 1024 == 0) + if ((n_buffers * (block_size / 1024)) % 1024 == 0) snprintf(repltok, sizeof(repltok), "%dMB", - (n_buffers * (cluster_block_size / 1024)) / 1024); + (n_buffers * (block_size / 1024)) / 1024); else snprintf(repltok, sizeof(repltok), "%dkB", - n_buffers * (cluster_block_size / 1024)); + n_buffers * (block_size / 1024)); conflines = replace_guc_value(conflines, "shared_buffers", repltok, false); @@ -1296,21 +1300,21 @@ setup_config(void) #if DEFAULT_BACKEND_FLUSH_AFTER > 0 snprintf(repltok, sizeof(repltok), "%dkB", - DEFAULT_BACKEND_FLUSH_AFTER * (cluster_block_size / 1024)); + DEFAULT_BACKEND_FLUSH_AFTER * (block_size / 1024)); conflines = replace_guc_value(conflines, "backend_flush_after", repltok, true); #endif #if DEFAULT_BGWRITER_FLUSH_AFTER > 0 snprintf(repltok, sizeof(repltok), "%dkB", - DEFAULT_BGWRITER_FLUSH_AFTER * (cluster_block_size / 1024)); + DEFAULT_BGWRITER_FLUSH_AFTER * (block_size / 1024)); conflines = replace_guc_value(conflines, "bgwriter_flush_after", repltok, true); #endif #if DEFAULT_CHECKPOINT_FLUSH_AFTER > 0 snprintf(repltok, sizeof(repltok), "%dkB", - DEFAULT_CHECKPOINT_FLUSH_AFTER * (cluster_block_size / 1024)); + DEFAULT_CHECKPOINT_FLUSH_AFTER * (block_size / 1024)); conflines = replace_guc_value(conflines, "checkpoint_flush_after", repltok, true); #endif @@ -1537,12 +1541,12 @@ bootstrap_template1(void) printfPQExpBuffer(&cmd, "\"%s\" --boot %s %s", backend_exec, boot_options, extra_options); appendPQExpBuffer(&cmd, " -X %d", wal_segment_size_mb * (1024 * 1024)); + appendPQExpBuffer(&cmd, " -b %d", block_size); if (data_checksums) appendPQExpBuffer(&cmd, " -k"); if (debug) appendPQExpBuffer(&cmd, " -d 5"); - PG_CMD_OPEN(cmd.data); for (line = bki_lines; *line != NULL; line++) @@ -2436,6 +2440,7 @@ usage(const char *progname) printf(_(" -A, --auth=METHOD default authentication method for local connections\n")); printf(_(" --auth-host=METHOD default authentication method for local TCP/IP connections\n")); printf(_(" --auth-local=METHOD default authentication method for local-socket connections\n")); + printf(_(" -b, --block-size=SIZE size of database blocks, in kilobytes\n")); printf(_(" [-D, --pgdata=]DATADIR location for this database cluster\n")); printf(_(" -E, --encoding=ENCODING set default encoding for new databases\n")); printf(_(" -g, --allow-group-access allow group read/execute on data directory\n")); @@ -3100,6 +3105,7 @@ main(int argc, char *argv[]) {"sync-only", no_argument, NULL, 'S'}, {"waldir", required_argument, NULL, 'X'}, {"wal-segsize", required_argument, NULL, 12}, + {"block-size", required_argument, NULL, 'b'}, {"data-checksums", no_argument, NULL, 'k'}, {"allow-group-access", no_argument, NULL, 'g'}, {"discard-caches", no_argument, NULL, 14}, @@ -3147,7 +3153,7 @@ main(int argc, char *argv[]) /* process command-line options */ - while ((c = getopt_long(argc, argv, "A:c:dD:E:gkL:nNsST:U:WX:", + while ((c = getopt_long(argc, argv, "A:b:c:dD:E:gkL:nNsST:U:WX:", long_options, &option_index)) != -1) { switch (c) @@ -3171,6 +3177,9 @@ main(int argc, char *argv[]) case 11: authmethodhost = pg_strdup(optarg); break; + case 'b': + str_block_size = pg_strdup(optarg); + break; case 'c': { char *buf = pg_strdup(optarg); @@ -3352,6 +3361,26 @@ main(int argc, char *argv[]) if (!IsValidWalSegSize(wal_segment_size_mb * 1024 * 1024)) pg_fatal("argument of %s must be a power of two between 1 and 1024", "--wal-segsize"); + if (str_block_size == NULL) + block_size = DEFAULT_BLOCK_SIZE; + else + { + char *endptr; + + /* check that the argument is a number */ + block_size = strtol(str_block_size, &endptr, 10); + + /* verify that the segment size is valid */ + if (endptr == str_block_size || *endptr != '\0') + pg_fatal("argument of --block-size must be a number"); + block_size *= 1024; + /* check for valid block_size; last is bitwise power of two check */ + if (!IsValidBlockSize(block_size)) + pg_fatal("argument of --block-size must be a power of 2 between 1 and 32"); + } + + BlockSizeInit(block_size); + get_restricted_token(); setup_pgdata(); @@ -3385,6 +3414,8 @@ main(int argc, char *argv[]) else printf(_("Data page checksums are disabled.\n")); + printf(_("Selected server block size: %d\n"), block_size); + if (pwprompt || pwfilename) get_su_pwd(); diff --git a/src/bin/initdb/meson.build b/src/bin/initdb/meson.build index 49743630aa..e2b8d88bff 100644 --- a/src/bin/initdb/meson.build +++ b/src/bin/initdb/meson.build @@ -31,6 +31,7 @@ tests += { 'env': {'with_icu': icu.found() ? 'yes' : 'no'}, 'tests': [ 't/001_initdb.pl', + 't/002_blocksize.pl', ], }, } diff --git a/src/bin/initdb/t/002_blocksize.pl b/src/bin/initdb/t/002_blocksize.pl new file mode 100644 index 0000000000..d8d4208fed --- /dev/null +++ b/src/bin/initdb/t/002_blocksize.pl @@ -0,0 +1,24 @@ + +# Copyright (c) 2023, PostgreSQL Global Development Group + + +use strict; +use warnings; +use Fcntl ':mode'; +use File::stat qw{lstat}; +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::Utils; +use Test::More; + +for my $blocksize (1,2,4,8,16,32) { + my $node1 = PostgreSQL::Test::Cluster->new('node' . $blocksize); + $node1->init(extra => ['--block-size='.($blocksize)] ); + $node1->start; + + is($node1->safe_psql('postgres',q{SELECT current_setting('block_size')}), + 1024 * $blocksize, "initdb with blocksize " . $blocksize . "k"); + + $node1->stop; +} + +done_testing(); diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index 1dc8efe0cb..b983c2c354 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -28,6 +28,7 @@ #include "access/xlog_internal.h" #include "backup/basebackup.h" #include "bbstreamer.h" +#include "common/blocksize.h" #include "common/compression.h" #include "common/file_perm.h" #include "common/file_utils.h" @@ -2735,6 +2736,12 @@ main(int argc, char **argv) if (!RetrieveWalSegSize(conn)) exit(1); + /* detect and set remote server's block size */ + if (!RetrieveBlockSize(conn)) + exit(1); + + BlockSizeInit(BlockSize); + /* Create pg_wal symlink, if required */ if (xlog_dir) { diff --git a/src/bin/pg_basebackup/pg_receivewal.c b/src/bin/pg_basebackup/pg_receivewal.c index d0a4079d50..e5f5c804ee 100644 --- a/src/bin/pg_basebackup/pg_receivewal.c +++ b/src/bin/pg_basebackup/pg_receivewal.c @@ -897,6 +897,10 @@ main(int argc, char **argv) /* determine remote server's xlog segment size */ if (!RetrieveWalSegSize(conn)) exit(1); + if (!RetrieveBlockSize(conn)) + exit(1); + + BlockSizeInit(BlockSize); /* * Don't close the connection here so that subsequent StreamLog() can diff --git a/src/bin/pg_basebackup/streamutil.c b/src/bin/pg_basebackup/streamutil.c index 75ab9e56f3..9ff0fd8e34 100644 --- a/src/bin/pg_basebackup/streamutil.c +++ b/src/bin/pg_basebackup/streamutil.c @@ -32,6 +32,7 @@ #define ERRCODE_DUPLICATE_OBJECT "42710" int WalSegSz; +int BlockSize; static bool RetrieveDataDirCreatePerm(PGconn *conn); @@ -332,6 +333,64 @@ RetrieveWalSegSize(PGconn *conn) return true; } +/* + * Use SHOW block_size since ControlFile is not accessible here. + */ +bool +RetrieveBlockSize(PGconn *conn) +{ + PGresult *res; + + /* check connection existence */ + Assert(conn != NULL); + + /* for previous versions set the default xlog seg size */ + if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_SHOW_CMD) + { + BlockSize = DEFAULT_BLOCK_SIZE; + return true; + } + + res = PQexec(conn, "SHOW block_size"); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + pg_log_error("could not send replication command \"%s\": %s", + "SHOW block_size", PQerrorMessage(conn)); + + PQclear(res); + return false; + } + if (PQntuples(res) != 1 || PQnfields(res) < 1) + { + pg_log_error("could not fetch block size: got %d rows and %d fields, expected %d rows and %d or more fields", + PQntuples(res), PQnfields(res), 1, 1); + + PQclear(res); + return false; + } + + /* fetch xlog value and unit from the result */ + if (sscanf(PQgetvalue(res, 0, 0), "%d", &BlockSize) != 1) + { + pg_log_error("block size could not be parsed"); + PQclear(res); + return false; + } + + PQclear(res); + + if (!IsValidBlockSize(BlockSize)) + { + pg_log_error(ngettext("Block size must be a power of two between 1k and 32k, but the remote server reported a value of %d byte", + "Block size must be a power of two between 1k and 32k, but the remote server reported a value of %d bytes", + BlockSize), + BlockSize); + return false; + } + + return true; +} + /* * RetrieveDataDirCreatePerm * diff --git a/src/bin/pg_basebackup/streamutil.h b/src/bin/pg_basebackup/streamutil.h index 268c163213..c97f9fa969 100644 --- a/src/bin/pg_basebackup/streamutil.h +++ b/src/bin/pg_basebackup/streamutil.h @@ -13,6 +13,7 @@ #define STREAMUTIL_H #include "access/xlogdefs.h" +#include "common/blocksize.h" #include "datatype/timestamp.h" #include "libpq-fe.h" #include "pqexpbuffer.h" @@ -25,6 +26,7 @@ extern char *dbport; extern char *dbname; extern int dbgetpassword; extern int WalSegSz; +extern int BlockSize; /* Connection kept global so we can disconnect easily */ extern PGconn *conn; @@ -56,6 +58,7 @@ extern bool GetSlotInformation(PGconn *conn, const char *slot_name, XLogRecPtr *restart_lsn, TimeLineID *restart_tli); extern bool RetrieveWalSegSize(PGconn *conn); +extern bool RetrieveBlockSize(PGconn *conn); extern TimestampTz feGetCurrentTimestamp(void); extern void feTimestampDifference(TimestampTz start_time, TimestampTz stop_time, long *secs, int *microsecs); diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index 0323bc4937..107ed1fd64 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -21,6 +21,7 @@ #include #include "access/xlog_internal.h" +#include "common/blocksize.h" #include "common/controldata_utils.h" #include "common/file_perm.h" #include "common/file_utils.h" @@ -231,14 +232,14 @@ scan_file(const char *fn, int segmentno) if (PageIsNew(buf.data)) continue; - csum = pg_checksum_page(buf.data, blockno + segmentno * RELSEG_SIZE); + csum = pg_checksum_page(buf.data, blockno + segmentno * RELSEG_SIZE, cluster_block_size); if (mode == PG_MODE_CHECK) { if (csum != header->pd_checksum) { if (ControlFile->data_checksum_version == PG_DATA_CHECKSUM_VERSION) - pg_log_error("checksum verification failed in file \"%s\", block %u: calculated checksum %X but block contains %X", - fn, blockno, csum, header->pd_checksum); + pg_log_error("checksum verification failed in file \"%s\", block %u, block_size: %u: calculated checksum %X but block contains %X", + fn, blockno, cluster_block_size, csum, header->pd_checksum); badblocks++; } } @@ -259,7 +260,7 @@ scan_file(const char *fn, int segmentno) header->pd_checksum = csum; /* Seek back to beginning of block */ - if (lseek(f, -cluster_block_size, SEEK_CUR) < 0) + if (lseek(f, -(int)cluster_block_size, SEEK_CUR) < 0) pg_fatal("seek failed for block %u in file \"%s\": %m", blockno, fn); /* Write block with checksum */ @@ -551,14 +552,6 @@ main(int argc, char *argv[]) if (ControlFile->pg_control_version != PG_CONTROL_VERSION) pg_fatal("cluster is not compatible with this version of pg_checksums"); - if (ControlFile->blcksz != cluster_block_size) - { - pg_log_error("database cluster is not compatible"); - pg_log_error_detail("The database cluster was initialized with block size %u, but pg_checksums was compiled with block size %u.", - ControlFile->blcksz, cluster_block_size); - exit(1); - } - /* * Check if cluster is running. A clean shutdown is required to avoid * random checksum failures caused by torn pages. Note that this doesn't @@ -580,6 +573,11 @@ main(int argc, char *argv[]) mode == PG_MODE_ENABLE) pg_fatal("data checksums are already enabled in cluster"); + if (IsValidBlockSize(ControlFile->blcksz)) + BlockSizeInit(ControlFile->blcksz); + else + pg_fatal("invalid cluster block size in control file"); + /* Operate on all files if checking or enabling checksums */ if (mode == PG_MODE_CHECK || mode == PG_MODE_ENABLE) { diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c index 84c65069aa..da9b21776f 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c @@ -49,6 +49,7 @@ #include "access/transam.h" #include "access/xlog.h" #include "access/xlog_internal.h" +#include "common/blocksize.h" #include "common/controldata_utils.h" #include "common/fe_memutils.h" #include "common/file_perm.h" @@ -76,6 +77,7 @@ static TimeLineID minXlogTli = 0; static XLogSegNo minXlogSegNo = 0; static int WalSegSz; static int set_wal_segsize; +static int set_blocksize = 0; static void CheckDataVersion(void); static bool read_controlfile(void); @@ -94,6 +96,7 @@ int main(int argc, char *argv[]) { static struct option long_options[] = { + {"block-size", required_argument, NULL, 'b'}, {"commit-timestamp-ids", required_argument, NULL, 'c'}, {"pgdata", required_argument, NULL, 'D'}, {"epoch", required_argument, NULL, 'e'}, @@ -138,10 +141,19 @@ main(int argc, char *argv[]) } - while ((c = getopt_long(argc, argv, "c:D:e:fl:m:no:O:u:x:", long_options, NULL)) != -1) + while ((c = getopt_long(argc, argv, "b:c:D:e:fl:m:no:O:u:x:", long_options, NULL)) != -1) { switch (c) { + case 'b': + errno = 0; + set_blocksize = strtol(optarg, &endptr, 10); + if (endptr == optarg || *endptr != '\0' || errno != 0) + pg_fatal("argument of --block-size must be a number"); + set_blocksize *= 1024; + if (!IsValidBlockSize(set_blocksize)) + pg_fatal("argument of --block-size must be a power of 2 between 1 and 32"); + break; case 'D': DataDir = optarg; break; @@ -391,6 +403,33 @@ main(int argc, char *argv[]) else WalSegSz = ControlFile.xlog_seg_size; + /* + * If a blocksize was specified, compare to existing ControlFile; if we + * are wrong, we won't be able to read the data. We will only want to set + * it if we guessed. + */ + if (set_blocksize == 0) + { + if (guessed) + pg_fatal("Cannot determine cluster block size; provide explicitly via --block-size"); + } + else + { + if (!guessed && set_blocksize != ControlFile.blcksz) + pg_fatal("Cannot change blocksize in cluster"); + + /* hope this is right, but by default we don't know; likely this is + * DEFAULT_BLOCK_SIZE */ + ControlFile.blcksz = set_blocksize; + } + + /* + * Set some dependent calculated fields stored in pg_control + */ + BlockSizeInit(ControlFile.blcksz); + ControlFile.toast_max_chunk_size = cluster_toast_max_chunk_size; + ControlFile.loblksize = cluster_loblksize; + if (log_fname != NULL) XLogFromFileName(log_fname, &minXlogTli, &minXlogSegNo, WalSegSz); @@ -614,6 +653,16 @@ read_controlfile(void) return false; } + /* return false if block size is not valid */ + if (!IsValidBlockSize(ControlFile.blcksz)) + { + pg_log_warning(ngettext("pg_control specifies invalid block size (%d byte); proceed with caution", + "pg_control specifies invalid block size (%d bytes); proceed with caution", + ControlFile.blcksz), + ControlFile.blcksz); + return false; + } + return true; } @@ -686,7 +735,7 @@ GuessControlValues(void) ControlFile.maxAlign = MAXIMUM_ALIGNOF; ControlFile.floatFormat = FLOATFORMAT_VALUE; - ControlFile.blcksz = cluster_block_size; + ControlFile.blcksz = DEFAULT_BLOCK_SIZE; ControlFile.relseg_size = RELSEG_SIZE; ControlFile.xlog_blcksz = XLOG_BLCKSZ; ControlFile.xlog_seg_size = DEFAULT_XLOG_SEG_SIZE; @@ -1131,6 +1180,7 @@ usage(void) printf(_("%s resets the PostgreSQL write-ahead log.\n\n"), progname); printf(_("Usage:\n %s [OPTION]... DATADIR\n\n"), progname); printf(_("Options:\n")); + printf(_(" --block-size=SIZE cluster block size, in bytes\n")); printf(_(" -c, --commit-timestamp-ids=XID,XID\n" " set oldest and newest transactions bearing\n" " commit timestamp (zero means no change)\n")); diff --git a/src/bin/pg_resetwal/t/002_corrupted.pl b/src/bin/pg_resetwal/t/002_corrupted.pl index 6d19a1efd5..84ed4e92b0 100644 --- a/src/bin/pg_resetwal/t/002_corrupted.pl +++ b/src/bin/pg_resetwal/t/002_corrupted.pl @@ -31,7 +31,7 @@ print $fh pack("x[$size]"); close $fh; command_checks_all( - [ 'pg_resetwal', '-n', $node->data_dir ], + [ 'pg_resetwal', '-b', '8', '-n', $node->data_dir ], 0, [qr/pg_control version number/], [ @@ -47,7 +47,7 @@ print $fh $data, pack("x[" . ($size - 16) . "]"); close $fh; command_checks_all( - [ 'pg_resetwal', '-n', $node->data_dir ], + [ 'pg_resetwal', '-b', '8', '-n', $node->data_dir ], 0, [qr/pg_control version number/], [ diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c index a28de44170..1dea3efb5a 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -27,6 +27,7 @@ #include #include "catalog/pg_tablespace_d.h" +#include "common/blocksize.h" #include "common/hashfn.h" #include "common/string.h" #include "datapagemap.h" diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index d77dc890c9..dd1606bb55 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -18,6 +18,7 @@ #include "access/xlog_internal.h" #include "catalog/catversion.h" #include "catalog/pg_control.h" +#include "common/blocksize.h" #include "common/controldata_utils.h" #include "common/file_perm.h" #include "common/restricted_token.h" @@ -340,6 +341,13 @@ main(int argc, char **argv) sanityChecks(); + if (IsValidBlockSize(ControlFile_source.blcksz) && + ControlFile_source.blcksz == ControlFile_target.blcksz) + BlockSizeInit(ControlFile_source.blcksz); + else + pg_fatal("cluster block sizes do not match or are invalid: %d", + ControlFile_source.blcksz); + /* * Usually, the TLI can be found in the latest checkpoint record. But if * the source server is just being promoted (or it's a standby that's diff --git a/src/bin/pg_upgrade/file.c b/src/bin/pg_upgrade/file.c index 3143cf28aa..db16a420ba 100644 --- a/src/bin/pg_upgrade/file.c +++ b/src/bin/pg_upgrade/file.c @@ -19,14 +19,17 @@ #include #endif + #include "access/visibilitymapdefs.h" +#include "common/blocksize.h" #include "common/file_perm.h" #include "pg_upgrade.h" +#undef cluster_block_size +#define cluster_block_size BlockSize #include "storage/bufpage.h" #include "storage/checksum.h" #include "storage/checksum_impl.h" - /* * cloneFile() * @@ -96,7 +99,7 @@ copyFile(const char *src, const char *dst, schemaName, relName, dst, strerror(errno)); /* copy in fairly large chunks for best efficiency */ -#define COPY_BUF_SIZE (50 * cluster_block_size) +#define COPY_BUF_SIZE (50 * BlockSize) buffer = (char *) pg_malloc(COPY_BUF_SIZE); @@ -187,7 +190,7 @@ rewriteVisibilityMap(const char *fromfile, const char *tofile, struct stat statbuf; /* Compute number of old-format bytes per new page */ - rewriteVmBytesPerPage = (cluster_block_size - SizeOfPageHeaderData) / 2; + rewriteVmBytesPerPage = (BlockSize - SizeOfPageHeaderData) / 2; if ((src_fd = open(fromfile, O_RDONLY | PG_BINARY, 0)) < 0) pg_fatal("error while copying relation \"%s.%s\": could not open file \"%s\": %s", @@ -220,7 +223,7 @@ rewriteVisibilityMap(const char *fromfile, const char *tofile, PageHeaderData pageheader; bool old_lastblk; - if ((bytesRead = read(src_fd, buffer.data, cluster_block_size)) != cluster_block_size) + if ((bytesRead = read(src_fd, buffer.data, BlockSize)) != BlockSize) { if (bytesRead < 0) pg_fatal("error while copying relation \"%s.%s\": could not read file \"%s\": %s", @@ -230,7 +233,7 @@ rewriteVisibilityMap(const char *fromfile, const char *tofile, schemaName, relName, fromfile); } - totalBytesRead += cluster_block_size; + totalBytesRead += BlockSize; old_lastblk = (totalBytesRead == src_filesize); /* Save the page header data */ @@ -293,10 +296,10 @@ rewriteVisibilityMap(const char *fromfile, const char *tofile, /* Set new checksum for visibility map page, if enabled */ if (new_cluster.controldata.data_checksum_version != 0) ((PageHeader) new_vmbuf.data)->pd_checksum = - pg_checksum_page(new_vmbuf.data, new_blkno); + pg_checksum_page(new_vmbuf.data, new_blkno, BlockSize); errno = 0; - if (write(dst_fd, new_vmbuf.data, cluster_block_size) != cluster_block_size) + if (write(dst_fd, new_vmbuf.data, BlockSize) != BlockSize) { /* if write didn't set errno, assume problem is no disk space */ if (errno == 0) diff --git a/src/bin/pg_upgrade/meson.build b/src/bin/pg_upgrade/meson.build index 12a97f84e2..06b26e54a5 100644 --- a/src/bin/pg_upgrade/meson.build +++ b/src/bin/pg_upgrade/meson.build @@ -1,5 +1,7 @@ # Copyright (c) 2022-2023, PostgreSQL Global Development Group +fs = import('fs') + pg_upgrade_sources = files( 'check.c', 'controldata.c', diff --git a/src/bin/pg_upgrade/pg_upgrade.c b/src/bin/pg_upgrade/pg_upgrade.c index 4562dafcff..6a64d8689a 100644 --- a/src/bin/pg_upgrade/pg_upgrade.c +++ b/src/bin/pg_upgrade/pg_upgrade.c @@ -63,6 +63,7 @@ static void setup(char *argv0, bool *live_check); ClusterInfo old_cluster, new_cluster; OSInfo os_info; +uint32 BlockSize; char *output_files[] = { SERVER_LOG_FILE, @@ -126,6 +127,8 @@ main(int argc, char **argv) check_cluster_compatibility(live_check); + BlockSize = old_cluster.controldata.blocksz; + check_and_dump_old_cluster(live_check); diff --git a/src/bin/pg_upgrade/pg_upgrade.h b/src/bin/pg_upgrade/pg_upgrade.h index 7afa96716e..fa36020ea3 100644 --- a/src/bin/pg_upgrade/pg_upgrade.h +++ b/src/bin/pg_upgrade/pg_upgrade.h @@ -336,7 +336,7 @@ extern UserOpts user_opts; extern ClusterInfo old_cluster, new_cluster; extern OSInfo os_info; - +extern uint32 BlockSize; /* check.c */ diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c index 059836f0e6..dc187a9f4d 100644 --- a/src/bin/pg_verifybackup/pg_verifybackup.c +++ b/src/bin/pg_verifybackup/pg_verifybackup.c @@ -841,8 +841,8 @@ parse_required_wal(verifier_context *context, char *pg_waldump_path, { char *pg_waldump_cmd; - pg_waldump_cmd = psprintf("\"%s\" --quiet --path=\"%s\" --timeline=%u --start=%X/%X --end=%X/%X\n", - pg_waldump_path, wal_directory, this_wal_range->tli, + pg_waldump_cmd = psprintf("\"%s\" --quiet --pgdata=\"%s\" --path=\"%s\" --timeline=%u --start=%X/%X --end=%X/%X\n", + pg_waldump_path, context->backup_directory, wal_directory, this_wal_range->tli, LSN_FORMAT_ARGS(this_wal_range->start_lsn), LSN_FORMAT_ARGS(this_wal_range->end_lsn)); fflush(NULL); diff --git a/src/bin/pg_waldump/pg_waldump.c b/src/bin/pg_waldump/pg_waldump.c index b3ad58ec66..9365a4c0f1 100644 --- a/src/bin/pg_waldump/pg_waldump.c +++ b/src/bin/pg_waldump/pg_waldump.c @@ -23,6 +23,8 @@ #include "access/xlogreader.h" #include "access/xlogrecord.h" #include "access/xlogstats.h" +#include "common/blocksize.h" +#include "common/controldata_utils.h" #include "common/fe_memutils.h" #include "common/file_perm.h" #include "common/file_utils.h" @@ -38,8 +40,10 @@ */ static const char *progname; +static char *pgdata; static int WalSegSz; +static int BlockSize; static volatile sig_atomic_t time_to_stop = false; static const RelFileLocator emptyRelFileLocator = {0, 0, 0}; @@ -531,7 +535,7 @@ XLogRecordSaveFPWs(XLogReaderState *record, const char *savepath) if (!file) pg_fatal("could not open file \"%s\": %m", filename); - if (fwrite(page, cluster_block_size, 1, file) != 1) + if (fwrite(page, BlockSize, 1, file) != 1) pg_fatal("could not write file \"%s\": %m", filename); if (fclose(file) != 0) @@ -801,6 +805,8 @@ main(int argc, char **argv) XLogRecPtr first_record; char *waldir = NULL; char *errormsg; + ControlFileData *ControlFile; + bool crc_ok_p; static struct option long_options[] = { {"bkp-details", no_argument, NULL, 'b'}, @@ -812,6 +818,7 @@ main(int argc, char **argv) {"help", no_argument, NULL, '?'}, {"limit", required_argument, NULL, 'n'}, {"path", required_argument, NULL, 'p'}, + {"pgdata", required_argument, NULL, 'D'}, {"quiet", no_argument, NULL, 'q'}, {"relation", required_argument, NULL, 'R'}, {"rmgr", required_argument, NULL, 'r'}, @@ -885,7 +892,7 @@ main(int argc, char **argv) goto bad_argument; } - while ((option = getopt_long(argc, argv, "bB:e:fF:n:p:qr:R:s:t:wx:z", + while ((option = getopt_long(argc, argv, "bB:D:e:fF:n:p:qr:R:s:t:wx:z", long_options, &optindex)) != -1) { switch (option) @@ -903,6 +910,9 @@ main(int argc, char **argv) config.filter_by_relation_block_enabled = true; config.filter_by_extended = true; break; + case 'D': + pgdata = pg_strdup(optarg); + break; case 'e': if (sscanf(optarg, "%X/%X", &xlogid, &xrecoff) != 2) { @@ -1106,6 +1116,23 @@ main(int argc, char **argv) } } + if (!pgdata) + pgdata = getenv("PGDATA"); + if (!pgdata || strlen(pgdata) == 0) + pg_fatal("Must provide data directory via -D or PGDATA envvar"); + + ControlFile = get_controlfile(pgdata, &crc_ok_p); + + if (!ControlFile) + pg_fatal("Could not locate control file"); + + BlockSize = ControlFile->blcksz; + + if (!IsValidBlockSize(BlockSize)) + pg_fatal("read invalid block size from control file"); + + BlockSizeInit(BlockSize); + if (config.save_fullpage_path != NULL) create_fullpage_directory(config.save_fullpage_path); diff --git a/src/bin/pg_waldump/t/001_basic.pl b/src/bin/pg_waldump/t/001_basic.pl index 029a0d0521..4b33fe45a6 100644 --- a/src/bin/pg_waldump/t/001_basic.pl +++ b/src/bin/pg_waldump/t/001_basic.pl @@ -66,6 +66,8 @@ wal_level=logical }); $node->start; +$ENV{PGDATA} = $node->data_dir; + my ($start_lsn, $start_walfile) = split /\|/, $node->safe_psql('postgres', q{SELECT pg_current_wal_insert_lsn(), pg_walfile_name(pg_current_wal_insert_lsn())}); $node->safe_psql('postgres', q{ diff --git a/src/bin/pg_waldump/t/002_save_fullpage.pl b/src/bin/pg_waldump/t/002_save_fullpage.pl index f0725805f2..d136324968 100644 --- a/src/bin/pg_waldump/t/002_save_fullpage.pl +++ b/src/bin/pg_waldump/t/002_save_fullpage.pl @@ -38,6 +38,8 @@ max_wal_senders = 4 }); $node->start; +$ENV{PGDATA} = $node->data_dir; + # Generate data/WAL to examine that will have full pages in them. $node->safe_psql( 'postgres', @@ -73,6 +75,7 @@ $node->command_ok( [ 'pg_waldump', '--quiet', '--save-fullpage', "$tmp_folder/raw", + '-D', $node->data_dir, '--relation', $relation, $walfile ], diff --git a/src/common/Makefile b/src/common/Makefile index 113029bf7b..da96f2dc8d 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -48,6 +48,7 @@ LIBS += $(PTHREAD_LIBS) OBJS_COMMON = \ archive.o \ base64.o \ + blocksize.o \ checksum_helper.o \ compression.o \ config_info.o \ diff --git a/src/common/blocksize.c b/src/common/blocksize.c new file mode 100644 index 0000000000..640b1516d2 --- /dev/null +++ b/src/common/blocksize.c @@ -0,0 +1,53 @@ +/*------------------------------------------------------------------------- + * + * blocksize.c + * This file contains methods to calculate various size constants for variable-sized blocks. + * + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/access/common/clustersizes.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "access/heaptoast.h" +#include "access/htup_details.h" +#include "access/itup.h" +#include "access/nbtree_int.h" +#include "common/blocksize.h" +#ifndef FRONTEND +#include "storage/freespace.h" +#endif + +PGDLLIMPORT BlockSizeIdent cluster_block_setting = BLOCK_SIZE_UNSET; + +/* + * This routine will calculate and cache the necessary constants. This should + * be called once very very early in the process (as soon as the native block + * size is known, so after reading ControlFile). + */ + +void +BlockSizeInit(Size rawblocksize) +{ + uint32 bits = 0; + Size blocksize = rawblocksize; + + Assert(IsValidBlockSize(rawblocksize)); + + // calculate max number of bits in the passed-in size + while (blocksize >>= 1) + bits++; + + // our smallest block size, 1k, is 2^10, and we want this to be 1 if initialized + cluster_block_setting = (BlockSizeIdent)(bits - 10) + 1; + + #ifndef FRONTEND + /* also setup the FreeSpaceMap internal sizing */ + FreeSpaceMapInit(); + #endif +} diff --git a/src/common/file_utils.c b/src/common/file_utils.c index ea74aa1eb5..d218510b23 100644 --- a/src/common/file_utils.c +++ b/src/common/file_utils.c @@ -540,7 +540,7 @@ pg_pwritev_with_retry(int fd, const struct iovec *iov, int iovcnt, off_t offset) ssize_t pg_pwrite_zeros(int fd, size_t size, off_t offset) { - static const PGIOAlignedBlock zbuffer = {{0}}; /* worth cluster_block_size */ + static const PGIOAlignedBlock zbuffer = {{0}}; /* worth MAX_BLOCK_SIZE */ void *zerobuf_addr = unconstify(PGIOAlignedBlock *, &zbuffer)->data; struct iovec iov[PG_IOV_MAX]; size_t remaining_size = size; @@ -558,10 +558,10 @@ pg_pwrite_zeros(int fd, size_t size, off_t offset) iov[iovcnt].iov_base = zerobuf_addr; - if (remaining_size < cluster_block_size) + if (remaining_size < sizeof(zbuffer.data)) this_iov_size = remaining_size; else - this_iov_size = cluster_block_size; + this_iov_size = sizeof(zbuffer.data); iov[iovcnt].iov_len = this_iov_size; remaining_size -= this_iov_size; diff --git a/src/common/meson.build b/src/common/meson.build index 53942a9a61..6d39671157 100644 --- a/src/common/meson.build +++ b/src/common/meson.build @@ -3,6 +3,7 @@ common_sources = files( 'archive.c', 'base64.c', + 'blocksize.c', 'checksum_helper.c', 'compression.c', 'controldata_utils.c', diff --git a/src/include/access/gist_private.h b/src/include/access/gist_private.h index 4c1dd74db7..939b8ef3f3 100644 --- a/src/include/access/gist_private.h +++ b/src/include/access/gist_private.h @@ -171,7 +171,7 @@ typedef struct GISTScanOpaqueData GistNSN curPageLSN; /* pos in the WAL stream when page was read */ /* In a non-ordered search, returnable heap items are stored here: */ - GISTSearchHeapItem pageData[cluster_block_size / sizeof(IndexTupleData)]; + GISTSearchHeapItem pageData[MAX_BLOCK_SIZE / sizeof(IndexTupleData)]; OffsetNumber nPageData; /* number of valid items in array */ OffsetNumber curPageData; /* next item to return */ MemoryContext pageDataCxt; /* context holding the fetched tuples, for diff --git a/src/include/access/hash.h b/src/include/access/hash.h index 4f6df08020..7bab4c0105 100644 --- a/src/include/access/hash.h +++ b/src/include/access/hash.h @@ -124,7 +124,7 @@ typedef struct HashScanPosData int lastItem; /* last valid index in items[] */ int itemIndex; /* current index in items[] */ - HashScanPosItem items[MaxIndexTuplesPerPage]; /* MUST BE LAST */ + HashScanPosItem items[MaxIndexTuplesPerPageLimit]; /* MUST BE LAST */ } HashScanPosData; #define HashScanPosIsPinned(scanpos) \ @@ -227,7 +227,7 @@ typedef HashScanOpaqueData *HashScanOpaque; * the block size boundary. So we use cluster_block_size to determine the maximum number * of bitmaps. */ -#define HASH_MAX_BITMAPS Min(cluster_block_size / 8, 1024) +#define HASH_MAX_BITMAPS Min(DEFAULT_BLOCK_SIZE / 8, 1024) #define HASH_SPLITPOINT_PHASE_BITS 2 #define HASH_SPLITPOINT_PHASES_PER_GRP (1 << HASH_SPLITPOINT_PHASE_BITS) diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index faf5026519..644c03145b 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -75,7 +75,7 @@ typedef struct HeapScanDescData /* these fields only used in page-at-a-time mode and for bitmap scans */ int rs_cindex; /* current tuple's index in vistuples */ int rs_ntuples; /* number of visible tuples on page */ - OffsetNumber rs_vistuples[MaxHeapTuplesPerPage]; /* their offsets */ + OffsetNumber rs_vistuples[MaxHeapTuplesPerPageLimit]; /* their offsets */ } HeapScanDescData; typedef struct HeapScanDescData *HeapScanDesc; diff --git a/src/include/access/heaptoast.h b/src/include/access/heaptoast.h index 3658856a60..1aaf08a166 100644 --- a/src/include/access/heaptoast.h +++ b/src/include/access/heaptoast.h @@ -20,10 +20,8 @@ /* * Find the maximum size of a tuple if there are to be N tuples per page. */ -#define MaximumBytesPerTuple(tuplesPerPage) \ - MAXALIGN_DOWN((cluster_block_size - \ - MAXALIGN(SizeOfPageHeaderData + (tuplesPerPage) * sizeof(ItemIdData))) \ - / (tuplesPerPage)) +#define MaximumBytesPerTuple(tuplesPerPage) CalcMaximumBytesPerTuple(cluster_block_size,tuplesPerPage) +BlockSizeDecl2(CalcMaximumBytesPerTuple); /* * These symbols control toaster activation. If a tuple is larger than @@ -45,7 +43,7 @@ */ #define TOAST_TUPLES_PER_PAGE 4 -#define toast_tuple_threshold MaximumBytesPerTuple(TOAST_TUPLES_PER_PAGE) +#define toast_tuple_threshold CalcMaximumBytesPerTuple(cluster_block_size,TOAST_TUPLES_PER_PAGE) #define cluster_toast_tuple_target toast_tuple_threshold @@ -58,7 +56,7 @@ */ #define TOAST_TUPLES_PER_PAGE_MAIN 1 -#define toast_tuple_target_main MaximumBytesPerTuple(TOAST_TUPLES_PER_PAGE_MAIN) +#define toast_tuple_target_main CalcMaximumBytesPerTuple(cluster_block_size,TOAST_TUPLES_PER_PAGE_MAIN) /* * If an index value is larger than TOAST_INDEX_TARGET, we will try to @@ -81,12 +79,10 @@ #define EXTERN_TUPLE_MAX_SIZE MaximumBytesPerTuple(EXTERN_TUPLES_PER_PAGE) -#define cluster_toast_max_chunk_size \ - (EXTERN_TUPLE_MAX_SIZE - \ - MAXALIGN(SizeofHeapTupleHeader) - \ - sizeof(Oid) - \ - sizeof(int32) - \ - VARHDRSZ) +#define TOAST_MAX_CHUNK_SIZE_LIMIT CalcToastMaxChunkSize(MAX_BLOCK_SIZE) +#define cluster_toast_max_chunk_size BlockSizeCalc(cluster_block_setting,CalcToastMaxChunkSize) +BlockSizeDecl(CalcToastMaxChunkSize); + /* ---------- * heap_toast_insert_or_update - diff --git a/src/include/access/htup_details.h b/src/include/access/htup_details.h index 005a530e0a..7acb40386b 100644 --- a/src/include/access/htup_details.h +++ b/src/include/access/htup_details.h @@ -18,6 +18,7 @@ #include "access/transam.h" #include "access/tupdesc.h" #include "access/tupmacs.h" +#include "common/blocksize.h" #include "storage/bufpage.h" #include "varatt.h" @@ -427,8 +428,8 @@ do { \ (tup)->t_choice.t_heap.t_field3.t_xvac = (xid); \ } while (0) -StaticAssertDecl(MaxOffsetNumber < SpecTokenOffsetNumber, - "invalid speculative token constant"); +/* StaticAssertDecl(MaxOffsetNumber < SpecTokenOffsetNumber, */ +/* "invalid speculative token constant"); */ #define HeapTupleHeaderIsSpeculative(tup) \ ( \ @@ -560,8 +561,10 @@ StaticAssertDecl(MaxOffsetNumber < SpecTokenOffsetNumber, * ItemIds and tuples have different alignment requirements, don't assume that * you can, say, fit 2 tuples of size MaxHeapTupleSize/2 on the same page. */ -#define MaxHeapTupleSize (cluster_block_size - MAXALIGN(SizeOfPageHeaderData + sizeof(ItemIdData))) +#define MaxHeapTupleSize BlockSizeCalc(cluster_block_setting,CalcMaxHeapTupleSize) +#define MaxHeapTupleSizeLimit CalcMaxHeapTupleSize(MAX_BLOCK_SIZE) #define MinHeapTupleSize MAXALIGN(SizeofHeapTupleHeader) +BlockSizeDecl(CalcMaxHeapTupleSize); /* * MaxHeapTuplesPerPage is an upper bound on the number of tuples that can @@ -574,9 +577,10 @@ StaticAssertDecl(MaxOffsetNumber < SpecTokenOffsetNumber, * pointers to this anyway, to avoid excessive line-pointer bloat and not * require increases in the size of work arrays. */ -#define MaxHeapTuplesPerPage \ - ((int) ((cluster_block_size - SizeOfPageHeaderData) / \ - (MAXALIGN(SizeofHeapTupleHeader) + sizeof(ItemIdData)))) + +#define MaxHeapTuplesPerPage BlockSizeCalc(cluster_block_setting,CalcMaxHeapTuplesPerPage) +#define MaxHeapTuplesPerPageLimit CalcMaxHeapTuplesPerPage(MAX_BLOCK_SIZE) +BlockSizeDecl(CalcMaxHeapTuplesPerPage); /* * MaxAttrSize is a somewhat arbitrary upper limit on the declared size of diff --git a/src/include/access/itup.h b/src/include/access/itup.h index 8a68b9405b..90fce7ea15 100644 --- a/src/include/access/itup.h +++ b/src/include/access/itup.h @@ -163,8 +163,8 @@ index_getattr(IndexTuple tup, int attnum, TupleDesc tupleDesc, bool *isnull) * estimated here, seemingly allowing one more tuple than estimated here. * But such a page always has at least MAXALIGN special space, so we're safe. */ -#define MaxIndexTuplesPerPage \ - ((int) ((cluster_block_size - SizeOfPageHeaderData) / \ - (MAXALIGN(sizeof(IndexTupleData) + 1) + sizeof(ItemIdData)))) +#define MaxIndexTuplesPerPage BlockSizeCalc(cluster_block_setting,CalcMaxIndexTuplesPerPage) +#define MaxIndexTuplesPerPageLimit CalcMaxIndexTuplesPerPage(MAX_BLOCK_SIZE) +BlockSizeDecl(CalcMaxIndexTuplesPerPage); #endif /* ITUP_H */ diff --git a/src/include/access/nbtree.h b/src/include/access/nbtree.h index 033624e9c3..1071b0569f 100644 --- a/src/include/access/nbtree.h +++ b/src/include/access/nbtree.h @@ -15,6 +15,7 @@ #define NBTREE_H #include "access/amapi.h" +#include "access/nbtree_int.h" #include "access/itup.h" #include "access/sdir.h" #include "access/tableam.h" @@ -25,208 +26,6 @@ #include "storage/bufmgr.h" #include "storage/shm_toc.h" -/* There's room for a 16-bit vacuum cycle ID in BTPageOpaqueData */ -typedef uint16 BTCycleId; - -/* - * BTPageOpaqueData -- At the end of every page, we store a pointer - * to both siblings in the tree. This is used to do forward/backward - * index scans. The next-page link is also critical for recovery when - * a search has navigated to the wrong page due to concurrent page splits - * or deletions; see src/backend/access/nbtree/README for more info. - * - * In addition, we store the page's btree level (counting upwards from - * zero at a leaf page) as well as some flag bits indicating the page type - * and status. If the page is deleted, a BTDeletedPageData struct is stored - * in the page's tuple area, while a standard BTPageOpaqueData struct is - * stored in the page special area. - * - * We also store a "vacuum cycle ID". When a page is split while VACUUM is - * processing the index, a nonzero value associated with the VACUUM run is - * stored into both halves of the split page. (If VACUUM is not running, - * both pages receive zero cycleids.) This allows VACUUM to detect whether - * a page was split since it started, with a small probability of false match - * if the page was last split some exact multiple of MAX_BT_CYCLE_ID VACUUMs - * ago. Also, during a split, the BTP_SPLIT_END flag is cleared in the left - * (original) page, and set in the right page, but only if the next page - * to its right has a different cycleid. - * - * NOTE: the BTP_LEAF flag bit is redundant since level==0 could be tested - * instead. - * - * NOTE: the btpo_level field used to be a union type in order to allow - * deleted pages to store a 32-bit safexid in the same field. We now store - * 64-bit/full safexid values using BTDeletedPageData instead. - */ - -typedef struct BTPageOpaqueData -{ - BlockNumber btpo_prev; /* left sibling, or P_NONE if leftmost */ - BlockNumber btpo_next; /* right sibling, or P_NONE if rightmost */ - uint32 btpo_level; /* tree level --- zero for leaf pages */ - uint16 btpo_flags; /* flag bits, see below */ - BTCycleId btpo_cycleid; /* vacuum cycle ID of latest split */ -} BTPageOpaqueData; - -typedef BTPageOpaqueData *BTPageOpaque; - -#define BTPageGetOpaque(page) ((BTPageOpaque) PageGetSpecialPointer(page)) - -/* Bits defined in btpo_flags */ -#define BTP_LEAF (1 << 0) /* leaf page, i.e. not internal page */ -#define BTP_ROOT (1 << 1) /* root page (has no parent) */ -#define BTP_DELETED (1 << 2) /* page has been deleted from tree */ -#define BTP_META (1 << 3) /* meta-page */ -#define BTP_HALF_DEAD (1 << 4) /* empty, but still in tree */ -#define BTP_SPLIT_END (1 << 5) /* rightmost page of split group */ -#define BTP_HAS_GARBAGE (1 << 6) /* page has LP_DEAD tuples (deprecated) */ -#define BTP_INCOMPLETE_SPLIT (1 << 7) /* right sibling's downlink is missing */ -#define BTP_HAS_FULLXID (1 << 8) /* contains BTDeletedPageData */ - -/* - * The max allowed value of a cycle ID is a bit less than 64K. This is - * for convenience of pg_filedump and similar utilities: we want to use - * the last 2 bytes of special space as an index type indicator, and - * restricting cycle ID lets btree use that space for vacuum cycle IDs - * while still allowing index type to be identified. - */ -#define MAX_BT_CYCLE_ID 0xFF7F - - -/* - * The Meta page is always the first page in the btree index. - * Its primary purpose is to point to the location of the btree root page. - * We also point to the "fast" root, which is the current effective root; - * see README for discussion. - */ - -typedef struct BTMetaPageData -{ - uint32 btm_magic; /* should contain BTREE_MAGIC */ - uint32 btm_version; /* nbtree version (always <= BTREE_VERSION) */ - BlockNumber btm_root; /* current root location */ - uint32 btm_level; /* tree level of the root page */ - BlockNumber btm_fastroot; /* current "fast" root location */ - uint32 btm_fastlevel; /* tree level of the "fast" root page */ - /* remaining fields only valid when btm_version >= BTREE_NOVAC_VERSION */ - - /* number of deleted, non-recyclable pages during last cleanup */ - uint32 btm_last_cleanup_num_delpages; - /* number of heap tuples during last cleanup (deprecated) */ - float8 btm_last_cleanup_num_heap_tuples; - - bool btm_allequalimage; /* are all columns "equalimage"? */ -} BTMetaPageData; - -#define BTPageGetMeta(p) \ - ((BTMetaPageData *) PageGetContents(p)) - -/* - * The current Btree version is 4. That's what you'll get when you create - * a new index. - * - * Btree version 3 was used in PostgreSQL v11. It is mostly the same as - * version 4, but heap TIDs were not part of the keyspace. Index tuples - * with duplicate keys could be stored in any order. We continue to - * support reading and writing Btree versions 2 and 3, so that they don't - * need to be immediately re-indexed at pg_upgrade. In order to get the - * new heapkeyspace semantics, however, a REINDEX is needed. - * - * Deduplication is safe to use when the btm_allequalimage field is set to - * true. It's safe to read the btm_allequalimage field on version 3, but - * only version 4 indexes make use of deduplication. Even version 4 - * indexes created on PostgreSQL v12 will need a REINDEX to make use of - * deduplication, though, since there is no other way to set - * btm_allequalimage to true (pg_upgrade hasn't been taught to set the - * metapage field). - * - * Btree version 2 is mostly the same as version 3. There are two new - * fields in the metapage that were introduced in version 3. A version 2 - * metapage will be automatically upgraded to version 3 on the first - * insert to it. INCLUDE indexes cannot use version 2. - */ -#define BTREE_METAPAGE 0 /* first page is meta */ -#define BTREE_MAGIC 0x053162 /* magic number in metapage */ -#define BTREE_VERSION 4 /* current version number */ -#define BTREE_MIN_VERSION 2 /* minimum supported version */ -#define BTREE_NOVAC_VERSION 3 /* version with all meta fields set */ - -/* - * Maximum size of a btree index entry, including its tuple header. - * - * We actually need to be able to fit three items on every page, - * so restrict any one item to 1/3 the per-page available space. - * - * There are rare cases where _bt_truncate() will need to enlarge - * a heap index tuple to make space for a tiebreaker heap TID - * attribute, which we account for here. - */ -#define BTMaxItemSize(page) \ - (MAXALIGN_DOWN((PageGetPageSize(page) - \ - MAXALIGN(SizeOfPageHeaderData + 3*sizeof(ItemIdData)) - \ - MAXALIGN(sizeof(BTPageOpaqueData))) / 3) - \ - MAXALIGN(sizeof(ItemPointerData))) -#define BTMaxItemSizeNoHeapTid(page) \ - MAXALIGN_DOWN((PageGetPageSize(page) - \ - MAXALIGN(SizeOfPageHeaderData + 3*sizeof(ItemIdData)) - \ - MAXALIGN(sizeof(BTPageOpaqueData))) / 3) - -/* - * MaxTIDsPerBTreePage is an upper bound on the number of heap TIDs tuples - * that may be stored on a btree leaf page. It is used to size the - * per-page temporary buffers. - * - * Note: we don't bother considering per-tuple overheads here to keep - * things simple (value is based on how many elements a single array of - * heap TIDs must have to fill the space between the page header and - * special area). The value is slightly higher (i.e. more conservative) - * than necessary as a result, which is considered acceptable. - */ -#define MaxTIDsPerBTreePage \ - (int) ((cluster_block_size - SizeOfPageHeaderData - sizeof(BTPageOpaqueData)) / \ - sizeof(ItemPointerData)) - -/* - * The leaf-page fillfactor defaults to 90% but is user-adjustable. - * For pages above the leaf level, we use a fixed 70% fillfactor. - * The fillfactor is applied during index build and when splitting - * a rightmost page; when splitting non-rightmost pages we try to - * divide the data equally. When splitting a page that's entirely - * filled with a single value (duplicates), the effective leaf-page - * fillfactor is 96%, regardless of whether the page is a rightmost - * page. - */ -#define BTREE_MIN_FILLFACTOR 10 -#define BTREE_DEFAULT_FILLFACTOR 90 -#define BTREE_NONLEAF_FILLFACTOR 70 -#define BTREE_SINGLEVAL_FILLFACTOR 96 - -/* - * In general, the btree code tries to localize its knowledge about - * page layout to a couple of routines. However, we need a special - * value to indicate "no page number" in those places where we expect - * page numbers. We can use zero for this because we never need to - * make a pointer to the metadata page. - */ - -#define P_NONE 0 - -/* - * Macros to test whether a page is leftmost or rightmost on its tree level, - * as well as other state info kept in the opaque data. - */ -#define P_LEFTMOST(opaque) ((opaque)->btpo_prev == P_NONE) -#define P_RIGHTMOST(opaque) ((opaque)->btpo_next == P_NONE) -#define P_ISLEAF(opaque) (((opaque)->btpo_flags & BTP_LEAF) != 0) -#define P_ISROOT(opaque) (((opaque)->btpo_flags & BTP_ROOT) != 0) -#define P_ISDELETED(opaque) (((opaque)->btpo_flags & BTP_DELETED) != 0) -#define P_ISMETA(opaque) (((opaque)->btpo_flags & BTP_META) != 0) -#define P_ISHALFDEAD(opaque) (((opaque)->btpo_flags & BTP_HALF_DEAD) != 0) -#define P_IGNORE(opaque) (((opaque)->btpo_flags & (BTP_DELETED|BTP_HALF_DEAD)) != 0) -#define P_HAS_GARBAGE(opaque) (((opaque)->btpo_flags & BTP_HAS_GARBAGE) != 0) -#define P_INCOMPLETE_SPLIT(opaque) (((opaque)->btpo_flags & BTP_INCOMPLETE_SPLIT) != 0) -#define P_HAS_FULLXID(opaque) (((opaque)->btpo_flags & BTP_HAS_FULLXID) != 0) - /* * BTDeletedPageData is the page contents of a deleted page */ @@ -892,7 +691,7 @@ typedef struct BTDedupStateData * are implicitly unchanged by deduplication pass). */ int nintervals; /* current number of intervals in array */ - BTDedupInterval intervals[MaxIndexTuplesPerPage]; + BTDedupInterval intervals[MaxIndexTuplesPerPageLimit]; } BTDedupStateData; typedef BTDedupStateData *BTDedupState; @@ -987,7 +786,7 @@ typedef struct BTScanPosData int lastItem; /* last valid index in items[] */ int itemIndex; /* current index in items[] */ - BTScanPosItem items[MaxTIDsPerBTreePage]; /* MUST BE LAST */ + BTScanPosItem items[MaxTIDsPerBTreePageLimit]; /* MUST BE LAST */ } BTScanPosData; typedef BTScanPosData *BTScanPos; diff --git a/src/include/access/nbtree_int.h b/src/include/access/nbtree_int.h new file mode 100644 index 0000000000..923cc5cb29 --- /dev/null +++ b/src/include/access/nbtree_int.h @@ -0,0 +1,223 @@ +/*------------------------------------------------------------------------- + * + * nbtree_int.h + * internal defs for header file for postgres btree access method implementation. + * + * + * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/access/nbtree_int.h + * + *------------------------------------------------------------------------- + */ +#ifndef NBTREE_INT_H +#define NBTREE_INT_H + +#include "storage/block.h" +#include "storage/bufpage.h" +#include "storage/itemptr.h" + +/* There's room for a 16-bit vacuum cycle ID in BTPageOpaqueData */ +typedef uint16 BTCycleId; + +/* + * BTPageOpaqueData -- At the end of every page, we store a pointer + * to both siblings in the tree. This is used to do forward/backward + * index scans. The next-page link is also critical for recovery when + * a search has navigated to the wrong page due to concurrent page splits + * or deletions; see src/backend/access/nbtree/README for more info. + * + * In addition, we store the page's btree level (counting upwards from + * zero at a leaf page) as well as some flag bits indicating the page type + * and status. If the page is deleted, a BTDeletedPageData struct is stored + * in the page's tuple area, while a standard BTPageOpaqueData struct is + * stored in the page special area. + * + * We also store a "vacuum cycle ID". When a page is split while VACUUM is + * processing the index, a nonzero value associated with the VACUUM run is + * stored into both halves of the split page. (If VACUUM is not running, + * both pages receive zero cycleids.) This allows VACUUM to detect whether + * a page was split since it started, with a small probability of false match + * if the page was last split some exact multiple of MAX_BT_CYCLE_ID VACUUMs + * ago. Also, during a split, the BTP_SPLIT_END flag is cleared in the left + * (original) page, and set in the right page, but only if the next page + * to its right has a different cycleid. + * + * NOTE: the BTP_LEAF flag bit is redundant since level==0 could be tested + * instead. + * + * NOTE: the btpo_level field used to be a union type in order to allow + * deleted pages to store a 32-bit safexid in the same field. We now store + * 64-bit/full safexid values using BTDeletedPageData instead. + */ + +typedef struct BTPageOpaqueData +{ + BlockNumber btpo_prev; /* left sibling, or P_NONE if leftmost */ + BlockNumber btpo_next; /* right sibling, or P_NONE if rightmost */ + uint32 btpo_level; /* tree level --- zero for leaf pages */ + uint16 btpo_flags; /* flag bits, see below */ + BTCycleId btpo_cycleid; /* vacuum cycle ID of latest split */ +} BTPageOpaqueData; + +typedef BTPageOpaqueData *BTPageOpaque; + +#define BTPageGetOpaque(page) ((BTPageOpaque) PageGetSpecialPointer(page)) + +/* Bits defined in btpo_flags */ +#define BTP_LEAF (1 << 0) /* leaf page, i.e. not internal page */ +#define BTP_ROOT (1 << 1) /* root page (has no parent) */ +#define BTP_DELETED (1 << 2) /* page has been deleted from tree */ +#define BTP_META (1 << 3) /* meta-page */ +#define BTP_HALF_DEAD (1 << 4) /* empty, but still in tree */ +#define BTP_SPLIT_END (1 << 5) /* rightmost page of split group */ +#define BTP_HAS_GARBAGE (1 << 6) /* page has LP_DEAD tuples (deprecated) */ +#define BTP_INCOMPLETE_SPLIT (1 << 7) /* right sibling's downlink is missing */ +#define BTP_HAS_FULLXID (1 << 8) /* contains BTDeletedPageData */ + +/* + * The max allowed value of a cycle ID is a bit less than 64K. This is + * for convenience of pg_filedump and similar utilities: we want to use + * the last 2 bytes of special space as an index type indicator, and + * restricting cycle ID lets btree use that space for vacuum cycle IDs + * while still allowing index type to be identified. + */ +#define MAX_BT_CYCLE_ID 0xFF7F + + +/* + * The Meta page is always the first page in the btree index. + * Its primary purpose is to point to the location of the btree root page. + * We also point to the "fast" root, which is the current effective root; + * see README for discussion. + */ + +typedef struct BTMetaPageData +{ + uint32 btm_magic; /* should contain BTREE_MAGIC */ + uint32 btm_version; /* nbtree version (always <= BTREE_VERSION) */ + BlockNumber btm_root; /* current root location */ + uint32 btm_level; /* tree level of the root page */ + BlockNumber btm_fastroot; /* current "fast" root location */ + uint32 btm_fastlevel; /* tree level of the "fast" root page */ + /* remaining fields only valid when btm_version >= BTREE_NOVAC_VERSION */ + + /* number of deleted, non-recyclable pages during last cleanup */ + uint32 btm_last_cleanup_num_delpages; + /* number of heap tuples during last cleanup (deprecated) */ + float8 btm_last_cleanup_num_heap_tuples; + + bool btm_allequalimage; /* are all columns "equalimage"? */ +} BTMetaPageData; + +#define BTPageGetMeta(p) \ + ((BTMetaPageData *) PageGetContents(p)) + +/* + * The current Btree version is 4. That's what you'll get when you create + * a new index. + * + * Btree version 3 was used in PostgreSQL v11. It is mostly the same as + * version 4, but heap TIDs were not part of the keyspace. Index tuples + * with duplicate keys could be stored in any order. We continue to + * support reading and writing Btree versions 2 and 3, so that they don't + * need to be immediately re-indexed at pg_upgrade. In order to get the + * new heapkeyspace semantics, however, a REINDEX is needed. + * + * Deduplication is safe to use when the btm_allequalimage field is set to + * true. It's safe to read the btm_allequalimage field on version 3, but + * only version 4 indexes make use of deduplication. Even version 4 + * indexes created on PostgreSQL v12 will need a REINDEX to make use of + * deduplication, though, since there is no other way to set + * btm_allequalimage to true (pg_upgrade hasn't been taught to set the + * metapage field). + * + * Btree version 2 is mostly the same as version 3. There are two new + * fields in the metapage that were introduced in version 3. A version 2 + * metapage will be automatically upgraded to version 3 on the first + * insert to it. INCLUDE indexes cannot use version 2. + */ +#define BTREE_METAPAGE 0 /* first page is meta */ +#define BTREE_MAGIC 0x053162 /* magic number in metapage */ +#define BTREE_VERSION 4 /* current version number */ +#define BTREE_MIN_VERSION 2 /* minimum supported version */ +#define BTREE_NOVAC_VERSION 3 /* version with all meta fields set */ + +/* + * Maximum size of a btree index entry, including its tuple header. + * + * We actually need to be able to fit three items on every page, + * so restrict any one item to 1/3 the per-page available space. + * + * There are rare cases where _bt_truncate() will need to enlarge + * a heap index tuple to make space for a tiebreaker heap TID + * attribute, which we account for here. + */ +#define BTMaxItemSize(page) \ + (MAXALIGN_DOWN((PageGetPageSize(page) - \ + MAXALIGN(SizeOfPageHeaderData + 3*sizeof(ItemIdData)) - \ + MAXALIGN(sizeof(BTPageOpaqueData))) / 3) - \ + MAXALIGN(sizeof(ItemPointerData))) +#define BTMaxItemSizeNoHeapTid(page) \ + MAXALIGN_DOWN((PageGetPageSize(page) - \ + MAXALIGN(SizeOfPageHeaderData + 3*sizeof(ItemIdData)) - \ + MAXALIGN(sizeof(BTPageOpaqueData))) / 3) + +/* + * MaxTIDsPerBTreePage is an upper bound on the number of heap TIDs tuples + * that may be stored on a btree leaf page. It is used to size the + * per-page temporary buffers. + * + * Note: we don't bother considering per-tuple overheads here to keep + * things simple (value is based on how many elements a single array of + * heap TIDs must have to fill the space between the page header and + * special area). The value is slightly higher (i.e. more conservative) + * than necessary as a result, which is considered acceptable. + */ +#define MaxTIDsPerBTreePage BlockSizeCalc(cluster_block_setting,CalcMaxTIDsPerBTreePage) +#define MaxTIDsPerBTreePageLimit CalcMaxTIDsPerBTreePage(MAX_BLOCK_SIZE) +BlockSizeDecl(CalcMaxTIDsPerBTreePage); + +/* + * The leaf-page fillfactor defaults to 90% but is user-adjustable. + * For pages above the leaf level, we use a fixed 70% fillfactor. + * The fillfactor is applied during index build and when splitting + * a rightmost page; when splitting non-rightmost pages we try to + * divide the data equally. When splitting a page that's entirely + * filled with a single value (duplicates), the effective leaf-page + * fillfactor is 96%, regardless of whether the page is a rightmost + * page. + */ +#define BTREE_MIN_FILLFACTOR 10 +#define BTREE_DEFAULT_FILLFACTOR 90 +#define BTREE_NONLEAF_FILLFACTOR 70 +#define BTREE_SINGLEVAL_FILLFACTOR 96 + +/* + * In general, the btree code tries to localize its knowledge about + * page layout to a couple of routines. However, we need a special + * value to indicate "no page number" in those places where we expect + * page numbers. We can use zero for this because we never need to + * make a pointer to the metadata page. + */ + +#define P_NONE 0 + +/* + * Macros to test whether a page is leftmost or rightmost on its tree level, + * as well as other state info kept in the opaque data. + */ +#define P_LEFTMOST(opaque) ((opaque)->btpo_prev == P_NONE) +#define P_RIGHTMOST(opaque) ((opaque)->btpo_next == P_NONE) +#define P_ISLEAF(opaque) (((opaque)->btpo_flags & BTP_LEAF) != 0) +#define P_ISROOT(opaque) (((opaque)->btpo_flags & BTP_ROOT) != 0) +#define P_ISDELETED(opaque) (((opaque)->btpo_flags & BTP_DELETED) != 0) +#define P_ISMETA(opaque) (((opaque)->btpo_flags & BTP_META) != 0) +#define P_ISHALFDEAD(opaque) (((opaque)->btpo_flags & BTP_HALF_DEAD) != 0) +#define P_IGNORE(opaque) (((opaque)->btpo_flags & (BTP_DELETED|BTP_HALF_DEAD)) != 0) +#define P_HAS_GARBAGE(opaque) (((opaque)->btpo_flags & BTP_HAS_GARBAGE) != 0) +#define P_INCOMPLETE_SPLIT(opaque) (((opaque)->btpo_flags & BTP_INCOMPLETE_SPLIT) != 0) +#define P_HAS_FULLXID(opaque) (((opaque)->btpo_flags & BTP_HAS_FULLXID) != 0) + +#endif diff --git a/src/include/access/spgist_private.h b/src/include/access/spgist_private.h index ea803eccd8..89102369c0 100644 --- a/src/include/access/spgist_private.h +++ b/src/include/access/spgist_private.h @@ -226,14 +226,14 @@ typedef struct SpGistScanOpaqueData TupleDesc reconTupDesc; /* if so, descriptor for reconstructed tuples */ int nPtrs; /* number of TIDs found on current page */ int iPtr; /* index for scanning through same */ - ItemPointerData heapPtrs[MaxIndexTuplesPerPage]; /* TIDs from cur page */ - bool recheck[MaxIndexTuplesPerPage]; /* their recheck flags */ - bool recheckDistances[MaxIndexTuplesPerPage]; /* distance recheck + ItemPointerData heapPtrs[MaxIndexTuplesPerPageLimit]; /* TIDs from cur page */ + bool recheck[MaxIndexTuplesPerPageLimit]; /* their recheck flags */ + bool recheckDistances[MaxIndexTuplesPerPageLimit]; /* distance recheck * flags */ - HeapTuple reconTups[MaxIndexTuplesPerPage]; /* reconstructed tuples */ + HeapTuple reconTups[MaxIndexTuplesPerPageLimit]; /* reconstructed tuples */ /* distances (for recheck) */ - IndexOrderByDistance *distances[MaxIndexTuplesPerPage]; + IndexOrderByDistance *distances[MaxIndexTuplesPerPageLimit]; /* * Note: using MaxIndexTuplesPerPage above is a bit hokey since diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h index b0fd338a00..3bc419ed78 100644 --- a/src/include/access/xlog_internal.h +++ b/src/include/access/xlog_internal.h @@ -391,6 +391,7 @@ extern void GetOldestRestartPoint(XLogRecPtr *oldrecptr, TimeLineID *oldtli); extern void XLogRecGetBlockRefInfo(XLogReaderState *record, bool pretty, bool detailed_format, StringInfo buf, uint32 *fpi_len); +extern uint32 ClusterBlockSize(void); /* * Exported for the functions in timeline.c and xlogarchive.c. Only valid diff --git a/src/include/c.h b/src/include/c.h index f156886798..2b0835b988 100644 --- a/src/include/c.h +++ b/src/include/c.h @@ -1123,7 +1123,7 @@ extern void ExceptionalCondition(const char *conditionName, ((var) = ((var) < 0) ? 1 : -(var)) /* - * Use this, not "char buf[cluster_block_size]", to declare a field or local variable + * Use this, not "char buf[MAX_BLOCK_SIZE]", to declare a field or local variable * holding a page buffer, if that page might be accessed as a page. Otherwise * the variable might be under-aligned, causing problems on alignment-picky * hardware. We include both "double" and "int64" in the union to ensure that @@ -1132,7 +1132,7 @@ extern void ExceptionalCondition(const char *conditionName, */ typedef union PGAlignedBlock { - char data[cluster_block_size]; + char data[MAX_BLOCK_SIZE]; double force_align_d; int64 force_align_i64; } PGAlignedBlock; @@ -1150,7 +1150,7 @@ typedef union PGIOAlignedBlock #ifdef pg_attribute_aligned pg_attribute_aligned(PG_IO_ALIGN_SIZE) #endif - char data[cluster_block_size]; + char data[MAX_BLOCK_SIZE]; double force_align_d; int64 force_align_i64; } PGIOAlignedBlock; diff --git a/src/include/common/blocksize.h b/src/include/common/blocksize.h new file mode 100644 index 0000000000..b5fe75ea7b --- /dev/null +++ b/src/include/common/blocksize.h @@ -0,0 +1,119 @@ +/*------------------------------------------------------------------------- + * + * blocksize.h + * defintions for cluster-specific limits/structure defs + * + * + * Copyright (c) 2023, PostgreSQL Global Development Group + * + * IDENTIFICATION: src/include/clustersizes.h + * + *------------------------------------------------------------------------- + */ + +#ifndef BLOCKSIZE_H +#define BLOCKSIZE_H + +#ifndef DEFAULT_BLOCK_SIZE +#define DEFAULT_BLOCK_SIZE 8192 +#endif + +#ifndef MIN_BLOCK_SIZE +#define MIN_BLOCK_SIZE 1024 +#endif + +#ifndef MAX_BLOCK_SIZE +#define MAX_BLOCK_SIZE 32*1024 +#endif + +#define IsValidBlockSize(size) ((size) >= MIN_BLOCK_SIZE && \ + (size) <= MAX_BLOCK_SIZE && \ + ((size)&((size)-1)) == 0) + +/* identifiers only */ +typedef enum { + BLOCK_SIZE_UNSET = 0, + BLOCK_SIZE_1K, + BLOCK_SIZE_2K, + BLOCK_SIZE_4K, + BLOCK_SIZE_8K, + BLOCK_SIZE_16K, + BLOCK_SIZE_32K, +} BlockSizeIdent; + +extern PGDLLIMPORT BlockSizeIdent cluster_block_setting; + +void BlockSizeInit(Size rawblocksize); +#define cluster_block_bits (cluster_block_setting+9) +#define cluster_block_size (1<> cluster_block_bits) + +/* Specific calculations' now parameterized sources */ + +/* originally in heaptoast.h */ + +#define CalcMaximumBytesPerTuple(blocksize,tuplesPerPage) \ + MAXALIGN_DOWN((blocksize - \ + MAXALIGN(SizeOfPageHeaderData + (tuplesPerPage) * sizeof(ItemIdData))) \ + / (tuplesPerPage)) + +#define CalcToastMaxChunkSize(blocksize) \ + (CalcMaximumBytesPerTuple(blocksize,EXTERN_TUPLES_PER_PAGE) - \ + MAXALIGN(SizeofHeapTupleHeader) - \ + sizeof(Oid) - \ + sizeof(int32) - \ + VARHDRSZ) + +/* originally in htup_details.h */ + +#define CalcMaxHeapTupleSize(size) (size - MAXALIGN(SizeOfPageHeaderData + sizeof(ItemIdData))) + +#define CalcMaxHeapTuplesPerPage(size) \ + ((int) (((size) - SizeOfPageHeaderData) / \ + (MAXALIGN(SizeofHeapTupleHeader) + sizeof(ItemIdData)))) + +/* originally in itup.h */ + +#define CalcMaxIndexTuplesPerPage(size) \ + ((int) ((size - SizeOfPageHeaderData) / \ + (MAXALIGN(sizeof(IndexTupleData) + 1) + sizeof(ItemIdData)))) + +/* originally in nbtree_int.h */ + +#define CalcMaxTIDsPerBTreePage(size) \ + (int) ((size - SizeOfPageHeaderData - sizeof(BTPageOpaqueData)) / \ + sizeof(ItemPointerData)) + +/* originally in bloom.h */ +#define CalcFreeBlockNumberElems(size) (MAXALIGN_DOWN(size - SizeOfPageHeaderData - MAXALIGN(sizeof(BloomPageOpaqueData)) \ + - MAXALIGN(sizeof(uint16) * 2 + sizeof(uint32) + sizeof(BloomOptions)) \ + ) / sizeof(BlockNumber)) + +#define BlockSizeDecl(calc) \ + static inline unsigned int _block_size_##calc(BlockSizeIdent bsi) { \ + switch(bsi){ \ + case BLOCK_SIZE_1K: return calc(1024); break; \ + case BLOCK_SIZE_2K: return calc(2048); break; \ + case BLOCK_SIZE_4K: return calc(4096); break; \ + case BLOCK_SIZE_8K: return calc(8192); break; \ + case BLOCK_SIZE_16K: return calc(16384); break; \ + case BLOCK_SIZE_32K: return calc(32768); break; \ + default: return 0;}} + +#define BlockSizeDecl2(calc) \ + static inline unsigned int _block_size_##calc(BlockSizeIdent bsi, unsigned int arg) { \ + switch(bsi){ \ + case BLOCK_SIZE_1K: return calc(1024,arg); break; \ + case BLOCK_SIZE_2K: return calc(2048,arg); break; \ + case BLOCK_SIZE_4K: return calc(4096,arg); break; \ + case BLOCK_SIZE_8K: return calc(8192,arg); break; \ + case BLOCK_SIZE_16K: return calc(16384,arg); break; \ + case BLOCK_SIZE_32K: return calc(32768,arg); break; \ + default: return 0;}} + +#define BlockSizeCalc(bsi,calc) _block_size_##calc(bsi) + + +#endif diff --git a/src/include/pg_config.h.in b/src/include/pg_config.h.in index d8a2985567..acac7282b1 100644 --- a/src/include/pg_config.h.in +++ b/src/include/pg_config.h.in @@ -21,14 +21,15 @@ /* The normal alignment of `short', in bytes. */ #undef ALIGNOF_SHORT -/* Size of a disk block --- this also limits the size of a tuple. You can set - it bigger if you need bigger tuples (although TOAST should reduce the need - to have large tuples, since fields can be spread across multiple tuples). - BLCKSZ must be a power of 2. The maximum possible value of BLCKSZ is - currently 2^15 (32768). This is determined by the 15-bit widths of the - lp_off and lp_len fields in ItemIdData (see include/storage/itemid.h). - Changing BLCKSZ requires an initdb. */ -#undef BLCKSZ +/* Default size of a disk block --- this also limits the size of a tuple. You + can set it bigger if you need bigger tuples (although TOAST should reduce + the need to have large tuples, since fields can be spread across multiple + tuples). DEFAULT_BLOCK_SIZE must be a power of 2. The maximum possible + value of DEFAULT_BLOCK_SIZE is currently 2^15 (32768). This is determined + by the 15-bit widths of the lp_off and lp_len fields in ItemIdData (see + include/storage/itemid.h). Changing DEFAULT_BLOCK_SIZE requires an + initdb. */ +#undef DEFAULT_BLOCK_SIZE /* Saved arguments from configure */ #undef CONFIGURE_ARGS @@ -650,16 +651,16 @@ #undef PTHREAD_CREATE_JOINABLE /* RELSEG_SIZE is the maximum number of blocks allowed in one disk file. Thus, - the maximum size of a single file is RELSEG_SIZE * BLCKSZ; relations bigger - than that are divided into multiple files. RELSEG_SIZE * BLCKSZ must be - less than your OS' limit on file size. This is often 2 GB or 4GB in a - 32-bit operating system, unless you have large file support enabled. By - default, we make the limit 1 GB to avoid any possible integer-overflow - problems within the OS. A limit smaller than necessary only means we divide - a large relation into more chunks than necessary, so it seems best to err - in the direction of a small limit. A power-of-2 value is recommended to - save a few cycles in md.c, but is not absolutely required. Changing - RELSEG_SIZE requires an initdb. */ + the maximum size of a single file is RELSEG_SIZE * cluster_block_size; + relations bigger than that are divided into multiple files. RELSEG_SIZE * + cluster_block_size must be less than your OS' limit on file size. This is + often 2 GB or 4GB in a 32-bit operating system, unless you have large file + support enabled. By default, we make the limit 1 GB to avoid any possible + integer-overflow problems within the OS. A limit smaller than necessary + only means we divide a large relation into more chunks than necessary, so + it seems best to err in the direction of a small limit. A power-of-2 value + is recommended to save a few cycles in md.c, but is not absolutely + required. Changing RELSEG_SIZE requires an initdb. */ #undef RELSEG_SIZE /* The size of `bool', as computed by sizeof. */ @@ -774,7 +775,7 @@ # endif #endif -/* Size of a WAL file block. This need have no particular relation to BLCKSZ. +/* Size of a WAL file block. This need have no particular relation to cluster_block_size. XLOG_BLCKSZ must be a power of 2, and if your system supports O_DIRECT I/O, XLOG_BLCKSZ must be a multiple of the alignment requirement for direct-I/O buffers, else direct I/O may fail. Changing XLOG_BLCKSZ requires an initdb. diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h index 0446bfb407..70bddf2c7b 100644 --- a/src/include/pg_config_manual.h +++ b/src/include/pg_config_manual.h @@ -233,6 +233,14 @@ */ #define PG_IO_ALIGN_SIZE 4096 +/* + * Blocksize-related constants + */ + +#define MIN_BLOCK_SIZE 1024 +#define MAX_BLOCK_SIZE 32768 +/* #define DEFAULT_BLOCK_SIZE 8192 */ + /* *------------------------------------------------------------------------ * The following symbols are for enabling debugging code, not for diff --git a/src/include/postgres.h b/src/include/postgres.h index 8a028ff789..4651389fc2 100644 --- a/src/include/postgres.h +++ b/src/include/postgres.h @@ -45,6 +45,7 @@ #include "c.h" #include "utils/elog.h" #include "utils/palloc.h" +#include "common/blocksize.h" /* ---------------------------------------------------------------- * Section 1: Datum type + support functions diff --git a/src/include/postgres_ext.h b/src/include/postgres_ext.h index 1ed3d694ee..240ad4e93b 100644 --- a/src/include/postgres_ext.h +++ b/src/include/postgres_ext.h @@ -70,5 +70,4 @@ typedef PG_INT64_TYPE pg_int64; #define PG_DIAG_SOURCE_LINE 'L' #define PG_DIAG_SOURCE_FUNCTION 'R' -#define cluster_block_size BLCKSZ #endif /* POSTGRES_EXT_H */ diff --git a/src/include/storage/bufpage.h b/src/include/storage/bufpage.h index 20d8d7277a..a997380707 100644 --- a/src/include/storage/bufpage.h +++ b/src/include/storage/bufpage.h @@ -475,15 +475,15 @@ do { \ PageIsVerifiedExtended(page, blkno, \ PIV_LOG_WARNING | PIV_REPORT_STAT) -/* - * Check that cluster_block_size is a multiple of sizeof(size_t). In - * PageIsVerifiedExtended(), it is much faster to check if a page is - * full of zeroes using the native word size. Note that this assertion - * is kept within a header to make sure that StaticAssertDecl() works - * across various combinations of platforms and compilers. - */ -StaticAssertDecl(cluster_block_size == ((cluster_block_size / sizeof(size_t)) * sizeof(size_t)), - "cluster_block_size has to be a multiple of sizeof(size_t)"); +/* /\* */ +/* * Check that cluster_block_size is a multiple of sizeof(size_t). In */ +/* * PageIsVerifiedExtended(), it is much faster to check if a page is */ +/* * full of zeroes using the native word size. Note that this assertion */ +/* * is kept within a header to make sure that StaticAssertDecl() works */ +/* * across various combinations of platforms and compilers. */ +/* *\/ */ +/* StaticAssertDecl(cluster_block_size == ((cluster_block_size / sizeof(size_t)) * sizeof(size_t)), */ +/* "cluster_block_size has to be a multiple of sizeof(size_t)"); */ extern void PageInit(Page page, Size pageSize, Size specialSize); extern bool PageIsVerifiedExtended(Page page, BlockNumber blkno, int flags); diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h index 4afd25a0af..bea1cbd5b6 100644 --- a/src/include/storage/checksum.h +++ b/src/include/storage/checksum.h @@ -19,6 +19,6 @@ * Compute the checksum for a Postgres page. The page must be aligned on a * 4-byte boundary. */ -extern uint16 pg_checksum_page(char *page, BlockNumber blkno); +extern uint16 pg_checksum_page(char *page, BlockNumber blkno, Size pagesize); #endif /* CHECKSUM_H */ diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h index 18951c36c7..e51e04ec4a 100644 --- a/src/include/storage/checksum_impl.h +++ b/src/include/storage/checksum_impl.h @@ -101,6 +101,7 @@ */ #include "storage/bufpage.h" +#include "common/blocksize.h" /* number of checksums to calculate in parallel */ #define N_SUMS 32 @@ -111,7 +112,7 @@ typedef union { PageHeaderData phdr; - uint32 data[cluster_block_size / (sizeof(uint32) * N_SUMS)][N_SUMS]; + uint32 data[MAX_BLOCK_SIZE / (sizeof(uint32) * N_SUMS)][N_SUMS]; } PGChecksummablePage; /* @@ -143,7 +144,7 @@ do { \ * (at least on 4-byte boundary). */ static uint32 -pg_checksum_block(const PGChecksummablePage *page) +pg_checksum_block(const PGChecksummablePage *page, Size blocksize) { uint32 sums[N_SUMS]; uint32 result = 0; @@ -151,13 +152,13 @@ pg_checksum_block(const PGChecksummablePage *page) j; /* ensure that the size is compatible with the algorithm */ - Assert(sizeof(PGChecksummablePage) == cluster_block_size); + Assert(cluster_block_size == blocksize); /* initialize partial checksums to their corresponding offsets */ memcpy(sums, checksumBaseOffsets, sizeof(checksumBaseOffsets)); /* main checksum calculation */ - for (i = 0; i < (uint32) (cluster_block_size / (sizeof(uint32) * N_SUMS)); i++) + for (i = 0; i < (uint32) (blocksize / (sizeof(uint32) * N_SUMS)); i++) for (j = 0; j < N_SUMS; j++) CHECKSUM_COMP(sums[j], page->data[i][j]); @@ -184,7 +185,7 @@ pg_checksum_block(const PGChecksummablePage *page) * checksum itself), and the page data. */ uint16 -pg_checksum_page(char *page, BlockNumber blkno) +pg_checksum_page(char *page, BlockNumber blkno, Size pagesize) { PGChecksummablePage *cpage = (PGChecksummablePage *) page; uint16 save_checksum; @@ -201,7 +202,7 @@ pg_checksum_page(char *page, BlockNumber blkno) */ save_checksum = cpage->phdr.pd_checksum; cpage->phdr.pd_checksum = 0; - checksum = pg_checksum_block(cpage); + checksum = pg_checksum_block(cpage, pagesize); cpage->phdr.pd_checksum = save_checksum; /* Mix in the block number to detect transposed pages */ diff --git a/src/include/storage/freespace.h b/src/include/storage/freespace.h index 9e1a85a141..4bbad4aaad 100644 --- a/src/include/storage/freespace.h +++ b/src/include/storage/freespace.h @@ -19,6 +19,7 @@ #include "utils/relcache.h" /* prototypes for public functions in freespace.c */ +extern void FreeSpaceMapInit(void); extern Size GetRecordedFreeSpace(Relation rel, BlockNumber heapBlk); extern BlockNumber GetPageWithFreeSpace(Relation rel, Size spaceNeeded); extern BlockNumber RecordAndGetPageWithFreeSpace(Relation rel, diff --git a/src/include/storage/large_object.h b/src/include/storage/large_object.h index 9fab334c7b..9515df5f05 100644 --- a/src/include/storage/large_object.h +++ b/src/include/storage/large_object.h @@ -67,7 +67,9 @@ typedef struct LargeObjectDesc * * NB: Changing cluster_loblksize requires an initdb. */ +#define CalcLOBLKSIZE(size) (size/4) #define cluster_loblksize (cluster_block_size / 4) +#define LOBLKSIZE_LIMIT CalcLOBLKSIZE(MAX_BLOCK_SIZE) /* * Maximum length in bytes for a large object. To make this larger, we'd diff --git a/src/include/storage/off.h b/src/include/storage/off.h index 7fbd45a084..bfcd0a431f 100644 --- a/src/include/storage/off.h +++ b/src/include/storage/off.h @@ -25,7 +25,9 @@ typedef uint16 OffsetNumber; #define InvalidOffsetNumber ((OffsetNumber) 0) #define FirstOffsetNumber ((OffsetNumber) 1) -#define MaxOffsetNumber ((OffsetNumber) (cluster_block_size / sizeof(ItemIdData))) +#define CalcMaxOffsetNumber(size) ((OffsetNumber) (size / sizeof(ItemIdData))) +#define MaxOffsetNumber ((OffsetNumber)CalcMaxOffsetNumber(cluster_block_size)) +#define MaxOffsetNumberLimit ((OffsetNumber)CalcMaxOffsetNumber(MAX_BLOCK_SIZE)) /* ---------------- * support macros diff --git a/src/include/utils/guc_tables.h b/src/include/utils/guc_tables.h index d5a0880678..a47b6b6c2d 100644 --- a/src/include/utils/guc_tables.h +++ b/src/include/utils/guc_tables.h @@ -298,6 +298,7 @@ extern bool ConfigOptionIsVisible(struct config_generic *conf); /* get the current set of variables */ extern struct config_generic **get_guc_variables(int *num_vars); +extern void update_dynamic_gucs(void); extern void build_guc_variables(void); /* search in enum options */ diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm index 9e05eb91b1..ead1d4e26f 100644 --- a/src/tools/msvc/Mkvcbuild.pm +++ b/src/tools/msvc/Mkvcbuild.pm @@ -135,7 +135,7 @@ sub mkvcbuild } our @pgcommonallfiles = qw( - archive.c base64.c checksum_helper.c compression.c + archive.c base64.c blocksize.c checksum_helper.c compression.c config_info.c controldata_utils.c d2s.c encnames.c exec.c f2s.c file_perm.c file_utils.c hashfn.c ip.c jsonapi.c keywords.c kwlookup.c link-canary.c md5_common.c percentrepl.c diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index f6a40ae730..d7717d2bbc 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -212,10 +212,10 @@ sub GenerateFiles ALIGNOF_PG_INT128_TYPE => undef, ALIGNOF_SHORT => 2, AC_APPLE_UNIVERSAL_BUILD => undef, - cluster_block_size => 1024 * $self->{options}->{blocksize}, CONFIGURE_ARGS => '"' . $self->GetFakeConfigure() . '"', DEF_PGPORT => $port, DEF_PGPORT_STR => qq{"$port"}, + DEFAULT_BLOCK_SIZE => 1024 * $self->{options}->{blocksize}, DLSUFFIX => '".dll"', ENABLE_GSS => $self->{options}->{gss} ? 1 : undef, ENABLE_NLS => $self->{options}->{nls} ? 1 : undef, diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 49a33c0387..42375cca63 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -893,7 +893,6 @@ Form_pg_ts_template Form_pg_type Form_pg_user_mapping FormatNode -FreeBlockNumberArray FreeListData FreePageBtree FreePageBtreeHeader -- 2.39.2